"""Parse the 'Predictions of tropical activity' table from each season's Wikipedia wikitext
(raw/wiki_YYYY.txt) into data/wiki_forecasts.csv: year, source, date_text, ns, h, mh, ref_url.
Cells are kept as text (e.g. '11-16'); interpretation happens in build_table.py."""
import re, csv, glob, os
def strip(s):
    s = re.sub(r'<ref[^>]*/>', '', s)
    s = re.sub(r'<ref[^>]*>.*?</ref>', '', s, flags=re.S)
    s = re.sub(r'\{\{(?:nowrap|small|sort|nobr)\|([^{}]*)\}\}', r'\1', s)
    s = re.sub(r'\{\{[^{}]*\}\}', '', s)
    s = re.sub(r'\[\[(?:[^|\]]*\|)?([^\]]*)\]\]', r'\1', s)
    s = re.sub(r'<[^>]+>', ' ', s)
    s = re.sub(r"'''?", '', s)
    s = s.replace('&ndash;', '–').replace('&nbsp;', ' ')
    return re.sub(r'\s+', ' ', s).strip()
def cells(row):
    # split a table row into cells; drop attribute prefix 'align="x"|'
    row = re.sub(r'<ref[^>]*>.*?</ref>', lambda m: m.group(0).replace('|', '¦').replace('\n', ' '), row, flags=re.S)
    row = re.sub(r'\{\{[^{}]*\}\}', lambda m: m.group(0).replace('|', '¦'), row)
    row = re.sub(r'\[\[[^\]]*\]\]', lambda m: m.group(0).replace('|', '¦'), row)
    out = []
    for line in row.split('\n'):
        line = line.strip()
        if not line.startswith('|') and not line.startswith('!'):
            if out and line: out[-1] += ' ' + line
            continue
        for c in re.split(r'\|\||!!', line[1:]):
            if '|' in c and '=' in c.split('|')[0]:
                c = c.split('|', 1)[1]
            out.append(c)
    return [c.replace('¦', '|') for c in out]
out = []
for fn in sorted(glob.glob('../raw/wiki_*.txt')):
    yr = int(re.search(r'(\d{4})', os.path.basename(fn)).group(1))
    t = open(fn).read()
    m = re.search(r'Predictions of (?:tropical )?activity', t)
    if not m: print(yr, 'no table'); continue
    start = t.rfind('{|', 0, m.start()); end = t.find('\n|}', m.start())
    tab = t[start:end]
    for row in tab.split('\n|-')[1:]:
        cs = cells(row)
        urls = re.findall(r'url=\s*(\S+?)\s*[|}]', row)
        cs_txt = [strip(c) for c in cs]
        cs_txt = [c for c in cs_txt if c != '' or True]
        out.append([yr, len(cs_txt)] + cs_txt[:7] + [urls[0] if urls else ''])
with open('../data/wiki_rows_raw.csv', 'w', newline='') as fh:
    csv.writer(fh).writerows(out)
print(len(out))