I am still new to javascript and d3 . I am trying to find the best way to combine multiple csv external files into d3 then do something with it. Currently I am using script like this below for one file.
d3.csv("file1.csv",funciton(error,data) {
// do something
}
Is there a way that I can concatenate file1.csv + file2.csv + .. file9.csv into one file something similar to this .. in d3 ?
d3.csv( concat("file1.csv","file2,csv",..file9.csv") ,function(error,data) {
// do something
}
Here is a solution just using d3. You can see it in action in this Plunkr.
The Javascript code is:
function multiCsv(files, callback) {
var results = [];
var error = "";
var filesLength = (files || []).length;
var callbackInvoked = false;
for (var i = 0; i < filesLength; i++) {
(function(url) {
d3.csv(url, function(data) {
if (data === null) {
error += "Error retrieving \"" + url + "\"\n";
} else {
results.push(data);
}
// all files retrieved or an error occurred
if (!callbackInvoked && (error || results.length === filesLength)) {
if (error) {
callback(error, null); // might want to send partial results here
} else {
callback(null, d3.merge(results));
}
callbackInvoked = true;
}
});
})(files[i]);
}
}
You would use it like so:
multiCsv(["file1.csv", "file2.csv", "file3.csv"], function (err, results) {
if (err) {
alert(err);
return;
}
var ul = document.createElement('ul');
for (var i = 0, len = results.length; i < len; i++) {
var li = document.createElement('li');
li.textContent = results[i].FirstName + ' ' + results[i].LastName + ', ' + results[i].Age;
ul.appendChild(li);
}
document.body.appendChild(ul);
});
(This just adds a <ul> to the page with the contents of the merged array).
I haven't comprehensively tested this function, so YMMV. But it worked for my simple test case in Chrome.
i would suggest to do it on the server where you are hosting the javascript, then serve a single file to d3. Given that all csv files are the same format, and are named sequentially like your example, here is an easy way to join them on unix command line:
head -q -n 1 file1.csv > concat.csv #OVERWRITE with line 1 (headers)
tail -q -n +2 file*.csv >> concat.csv #APPEND lines 2+ from all matching files
edit, brute force clientside way, with jquery:
var remaining = 0;
var collection = [];
$(document).ready(concatCSVs)
function concatCSVs(){
var uris = ['csv1.csv', 'csv2.csv'];
remaining = uris.length;
uris.forEach(function(uri){
getCSV(uri, collector)
})
}
function getCSV(uri, callback){
$.ajax(uri, {
success: callback
})
}
function collector(data){
if(remaining == 0){
throw new Error('Got more results than expected')
}
remaining -= 1;
collection.push(data);
if(remaining == 0){ // Finished!
d3init(collection);
}
}
function d3init(collection){
console.log('Proceeding to load d3 with');
console.log(collection);
}
Splitting by lines and isolating the header row from content rows is left as an exercise for the reader
