import pandas as pd
import json
import requests
import qgrid
import datetime as dt
qgrid.nbinstall(overwrite=True)

qgrid.set_defaults(remote_js=True, show_toolbar=True)

urls = list(range(1, 91))
urls.insert(0, 'index')

threads = []
for u in urls:
    try:
        r = requests.get('http://2ch.hk/dr/' + str(u) + '.json')
        j = json.loads(r.text)
        for thread in j['threads']:
            threads.append([thread['posts'][0]['num'], thread['posts'][0]['name'], thread['posts'][0]['subject'], 
                            thread['posts'][0]['trip'], 
                            thread['posts'][0]['timestamp'], thread['posts'][-1]['timestamp'], thread['posts_count']])
    except Exception:
        break

data = pd.DataFrame(threads, columns = ['num', 'name', 'subject', 'trip', 'open_date', 'last_date', 'posts'])

data['open'] = data['open_date'].apply(lambda x: pd.to_datetime(dt.datetime.fromtimestamp(x)))
data['last'] = data['last_date'].apply(lambda x: pd.to_datetime(dt.datetime.fromtimestamp(x)))

data = data.set_index('num')
data.index = data.index.astype(int)
data['count'] = 1
data = data.drop(['open_date', 'last_date', 'count'], axis = 1)
data['duration'] = (data['last'] - data['open']).dt.days
data = data.sort_values('open')

qgrid.show_grid(data)