#!/bin/bash

#  press_scraper.sh - scrape and collate bitcoin press articles, output csv.
#  usage            - ./press_scraper.sh

# This program is free software. It comes without any warranty, to
# the extent permitted by applicable law. You can redistribute it
# and/or modify it under the terms of the Do What The Fuck You Want
# To Public License, Version 2, as published by Sam Hocevar. See
# http://sam.zoy.org/wtfpl/COPYING for more details.

total_articles=1760
decrement=40
tempfile=$(mktemp)
outfile=press_articles.csv

[ -f $tempfile ] || { echo "Error: Could not make temporary file. Exiting..."; \
  exit 1 ; }

function scrape {
  curl "$1" | sed -rn 's#.*<span id="msg_[0-9]+"><a href="https://bitcointalk\.org/index\.php\?topic=[0-9]+(\.0)?">([0-9]{4}-[0-1][0-9]-[0-3][0-9]).*</a></span>#\2#p' ;
}

for ((x=total_articles; x>40; x-=decrement))
do
  scrape "https://bitcointalk.org/index.php?board=77.$x" >> $tempfile
  sleep 5 # This is here just to be kind to the server, remove for speedup.
done

scrape "https://bitcointalk.org/index.php?board=77" >> $tempfile

sort $tempfile | uniq -c | sed -r 's/^ *([0-9]+) (.*)/\1,\2/' >$outfile

reset
clear
set xdata time
set format x "%Y-%m-%d"
set timefmt "%Y-%m-%d"
set datafile separator ","
set style fill solid noborder
set xtics rotate by -90 out nomirror 604800
set ytics out nomirror
set grid ytics
set ylabel "Press hits/day"
set xrange ["2012-04-07":"2013-04-26"]
set yrange [0:*]
set boxwidth 43200 absolute
set datafile separator ","
set term pngcairo truecolor font "Arial,11" size 1200,1200
set output "press_hits.png"
plot "press_articles.csv" using 2:1 with boxes ti "Press Article Frequency" lt 1 linecolor rgb "#FF0000"