import urllib2
import json
import csv
import time
import datetime

API_FETCH_URL = 'https://data.mtgox.com/api/2/BTCUSD/money/trades/fetch?since='

# mtgox API varname: postgres DB varname (from BigQuery dump)
VAR_MAP = {
    'price_currency': 'currency__',
    'trade_type': 'type',
    'price_int': 'price',
    'item': 'item',
    'primary': 'single_market',
    'tid': 'money_trade__',
    'amount_int': 'amount',
    'date': 'date',
    'properties': 'properties'
}
VARS = [var for var in VAR_MAP.keys()]

def process_date(timestamp):
    return datetime.datetime.utcfromtimestamp(timestamp).isoformat(' ')

def fetch_trades(last_id= None, outfile= '', limit=False, noise=False):
    counter = 0

    with open(outfile, 'wb') as f:
        # create CSV writer object
        writer = csv.writer(f)

        # add header of varnames at top of file
        writer.writerow([VAR_MAP[var] for var in VARS])

        while True:
            # pause for 2 seconds between API calls to prevent getting banned by anti-DDOS
            time.sleep(2)

            # fetch trades after the most recent trade id, using mtgox API
            page_data = urllib2.urlopen(API_FETCH_URL + last_id)
            # read response from urlopen GET request
            json_response = page_data.read()
            # decode JSON data into python dictionary
            response = json.loads(json_response)

            if response['result'] == 'success' and len(response['data']) > 0:
                trades = response['data']

                if noise:
                    print 'Batch %04d -- ?since= %s, num trades: %d' % (counter + 1, last_id, len(trades))

                # write each trade as a separate line, using only trade values for the vars in the list VARS
                # for date, convert from timestamp into ISO str format (UTC) to match date column in postgres DB
                writer.writerows([[trade[var] if var != 'date' else process_date(trade[var]) for var in VARS]
                                  for trade in trades])

                # set last_id to the tid of the last trade, so we can fetch next batch of trades
                last_id = trades[-1]['tid']

                counter = counter + 1
                # if limit parameter is in use, then only do as many batches as specified
                if limit != False and counter >= limit:
                    break
            else:
                print '\n**********'
                if response['result'] == 'success':
                    print 'SCRIPT END -- last trade reached'
                else:
                    print 'SCRIPT END -- API call failed'
                print '**********'
                break

if __name__ == '__main__':
    username = 'name'
    fileame = 'mtgox_recent_trades.csv'
    # id of last trade from BigQuery dump
    last_dump_trade_id = '1369319572499520'
    outfile = 'c:\\users\\%s\documents\\data\\mtgox\\%s' % (username, filename)

    fetch_trades(last_id= last_dump_trade_id, outfile= outfile, noise= True)