#!/usr/bin/env python3
"""Count recorded motifs in a pinned Berezkin catalogue, without using stale sums.

Usage: python3 count-motifs.py /path/to/berezkin_new.csv > motif-counts.json
Source (tab-delimited despite .csv extension):
https://raw.githubusercontent.com/macleginn/mythology-queries/5d055aa21d2a691b1a11ffd458c7194f1131ce50/data/berezkin_new.csv

Area 1 is the source's sub-Saharan grouping. All other rows form the comparator,
including North Africa and Madagascar. This is a descriptive count, not a
population-weighted or documentation-adjusted estimate of folkloric diversity.
"""
import csv
import hashlib
import json
import statistics
import sys
from pathlib import Path

path = Path(sys.argv[1])
with path.open(encoding='utf-8-sig', newline='') as stream:
    reader = csv.DictReader(stream, delimiter='\t')
    rows = list(reader)
    fields = reader.fieldnames
assert fields[:12] == ['idn', 'groups', 'latit', 'longit', 'sum', 'cosm_mot',
                      'areas1', 'AREA2', 'areas3', 'lang1', 'lang2', 'lang3']
traits = fields[12:]
assert (len(rows), len(traits)) == (926, 2138)
assert all(row[column] in ('0', '1') for row in rows for column in traits)
counts = [sum(int(row[column]) for column in traits) for row in rows]

def summarize(indices):
    values = [counts[i] for i in indices]
    return {'traditions': len(values), 'median_recorded_motifs': statistics.median(values),
            'mean_recorded_motifs': statistics.mean(values), 'minimum': min(values),
            'maximum': max(values)}

output = {
    'source_commit': '5d055aa21d2a691b1a11ffd458c7194f1131ce50',
    'source_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
    'matrix_dimensions': [len(rows), len(traits)],
    'count_rule': 'Sum all 2138 binary motif cells in each row; do not use sum metadata.',
    'sub_saharan_area_1': summarize([i for i, row in enumerate(rows) if row['areas1'] == '1']),
    'all_other_areas': summarize([i for i, row in enumerate(rows) if row['areas1'] != '1']),
    'all_source_areas': {
        key: summarize([i for i, row in enumerate(rows) if row['areas1'] == key])
        for key in sorted({row['areas1'] for row in rows}, key=int)
    },
    'stale_sum_metadata_rows': sum(count != int(row['sum']) for row, count in zip(rows, counts)),
    'limitations': [
        'A zero means unrecorded in this catalogue, not verified cultural absence.',
        'These are counts of selected recurrent motifs, not all stories or cultural sophistication.',
        'Tradition units and documentation effort vary; samples are not random.',
        'Richness within rows does not measure deep divergence between lineages.',
        'No significance test or prehistoric homeland inference is made.'
    ]
}
print(json.dumps(output, ensure_ascii=False, indent=2))