Hi
I just finished my history export code. Took me a while to get it working, I never did javascript before, so used a lot from stackexchange
You can inspect the script (you should) to see it's legit
I don't have the time to make code to process/filter the data, so that's up to you.
-------------updated:--------------
what it does:
convert a chosen section of your bitfinex history to cvs and a json-encoded array.
Usage:
-go to
https://www.bitfinex.com/ , if you use
https://bitfinex.com/ you need to change that in the code.
-paste the code to your browsers (I advice firefox) scratchpad (for firefox : shift+F4) (for chrome : control+shift+j) and run.
then:
Fill in the form. Click 'click me' ONCE to activate. The progress bar only works for firefox.
code::
function makeTableHTML(myArray) {
//http://stackoverflow.com/a/15164796
var result = "<table border=1>";
for (var i = 0; i < myArray.length; i++) {
result += "<tr>";
for (var j = 0; j < myArray[i].length; j++) {
result += "<td>" + myArray[i][j] + "</td>";
}
result += "</tr>";
}
result += "</table>";
return result;
}
function getTable(txt) {
//convert string to dom-object-thing, from http://stackoverflow.com/a/3104234
var tempDiv = document.createElement('div');
tempDiv.innerHTML = txt;
//find the table
var dataSet = tempDiv.getElementsByClassName("history table table-bordered table-striped")[0];
return dataSet.outerHTML;
}
// http://stackoverflow.com/a/9579792
function tableToArray(table) {
//convert string to dom-object-thing, from http://stackoverflow.com/a/3104234 and http://stackoverflow.com/a/18795833 for learning me how to use jq
var tempDiv = document.createElement('div');
tempDiv.innerHTML = table;
table = tempDiv;
var myTableArray = [];
$(table).find('tr').each(function () {
var arrayOfThisRow = [];
var tableData = $(this).find('td');
if (tableData.length > 0) {
tableData.each(function () {
arrayOfThisRow.push($(this).text().replace(/^\s+/,"").replace(/\s+$/,""));
});
myTableArray.push(arrayOfThisRow);
}
});
return (myTableArray);
}
//from http://stackoverflow.com/a/4033310
function httpGet(theUrl) {
var xmlHttp = null;
xmlHttp = new XMLHttpRequest();
xmlHttp.open("GET", theUrl, false);
xmlHttp.send(null);
return xmlHttp.responseText;
}
function getHistoryData(page, choice) {
var html = httpGet('https://www.bitfinex.com/' + choice + '?page=' + page);
var count = html.match(/cloudflare/g);
if (count !== null) {
alert("cloudflare page was loaded... Refreshing page and try again");
location.reload();
}
var a = getTable(html).replace(/\n/g,"").replace(/ {2,}/g, ' ');
var b = new Array();
b = tableToArray(a);
return b;
}
function getHistory(min, max, choice) {
var coll = [];
for (; min <= max; min++) {
var a = getHistoryData(min, choice);
updateProgress();
for (var i = 0; i < a.length; i++) {
coll[coll.length] = a[i];
}
}
return coll;
}
function addChoice() {
var newHTML = document.createElement('div');
newHTML.innerHTML = '----------------------------------------------------<br/><H3><b>READ-ME-FIRST</b></H3>If your history-page-count of your chosen section is high, you should consider splitting the scraping into batches of eg 25 (just a guess), because the script could get issues with the cloudflare-protection. The output will be a box with a json encoded array and the box below cvs. You should make sure no trades or interest payments happen when the parser is active, so I advice to cancel all trades and to not run this script between 01.00 and 02:00 GMT. <b>You should refresh this page if you want to scrape your next batch/section</b> -Wassupia<br/><br/>Note: the progress bar on Google Chrome does not work, click "Click Me" only once and wait!<br/><br/>updated on: March 11<br/> <br/> whiskey fund: 16pnBWFqyM5gCfMjTtpWYEv1hgVthehQEA<br/> <br/> <button type="button" onClick="go()">Click Me!</button><br/>History section<select id="choice" > <option value="account/ledger/usd">USD</option> <option value="account/ledger/btc">BTC</option> <option value="account/ledger/ltc">LTC</option> <option value="orders">Orders</option> <option value="offers">Offers</option> <option value="account/trades">Trades</option> <option value="account/unused_swaps">Unused swaps</option> <option value="account/margin_swaps">Margin swaps</option> </select>First page (>0) <input type="text" id="FirstPage" value="1">Last page (>0)<input type="text" id="LastPage" value="LastPage">CSVseparator<select id="CSVseparator"><option value=";">; (semicolon)</option> <option value=",">, (comma)</option><option value="|">| (vertical bar)</option> </select><br/><br/>';
document.body.appendChild(newHTML);
window.scrollTo(0,document.body.scrollHeight);
}
function printD(txt, message) {
var newHTML = document.createElement('div');
newHTML.innerHTML = '<br/><p>' + message + '</p><textarea rows="50" cols="50" onclick="this.focus();this.select()" readonly="readonly">' + txt + '</textarea> <br/>';
document.body.appendChild(newHTML);
window.scrollTo(0,document.body.scrollHeight);
}
function addProgressB(){
var newHTML = document.createElement('div');
newHTML.innerHTML = '<br/><p>progress:</p><input type="text" name="desc" id="uprogress" value="0" ></input>';
document.body.appendChild(newHTML);
window.scrollTo(0,document.body.scrollHeight);
}
function updateProgress()
{
document.getElementById("uprogress").value = 1 + parseInt(document.getElementById("uprogress").value);
}
function go() {
var min = document.getElementById("FirstPage").value;
var max = document.getElementById("LastPage").value;
var choice = document.getElementById("choice").value;
var separator = document.getElementById("CSVseparator").value;
if (max < 1 || min < 1){alert("firstpage and lastpage can't be less than 1");return '';}
if (max == "LastPage"){ alert('Fill in LastPage'); return ''; }
addProgressB();
var history = getHistory(min, max, choice);
printD(JSON.stringify(history), "jsonarray");
var csv = "";
for (var i = 0; i < history.length; i++) {
csv += history[i].join(separator) + "\r\n";
}
printD(csv, "csv");
}
addChoice();
edit: meant shift+ F4, not altF4 lol
edit 30 dec: spelling and return error if form not filled
edit March 11 2014: updated script with latest history sections, added cvs separator choise + readme update
