klz-cables.com/scripts/migrate-analytics-to-umami.py

#!/usr/bin/env python3
"""
Migrate Independent Analytics data to Umami format
"""

import csv
import json
import argparse
import uuid
import random
from datetime import datetime, timedelta
import sys

def parse_view_duration(duration_str):
    """Convert view duration from 'X:XX' format to seconds"""
    if not duration_str or duration_str == '-':
        return 0
    
    parts = duration_str.split(':')
    if len(parts) == 2:
        return int(parts[0]) * 60 + int(parts[1])
    elif len(parts) == 3:
        return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
    return 0

def convert_to_umami_format(csv_file, output_file, site_id="your-site-id"):
    """
    Convert Independent Analytics CSV to Umami import format
    
    Umami expects data in this format for API import:
    {
        "website_id": "uuid",
        "hostname": "example.com",
        "path": "/path",
        "referrer": "",
        "event_name": null,
        "pageview": true,
        "session": true,
        "duration": 0,
        "created_at": "2024-01-01T00:00:00.000Z"
    }
    """
    
    umami_records = []
    
    with open(csv_file, 'r', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        
        for row in reader:
            # Skip 404 pages and empty entries
            if row.get('Page Type') == '404' or not row.get('URL'):
                continue
            
            # Extract data
            title = row.get('Title', '')
            url = row.get('URL', '/')
            visitors = int(row.get('Visitors', 0))
            views = int(row.get('Views', 0))
            view_duration = parse_view_duration(row.get('View Duration', '0:00'))
            bounce_rate = float(row.get('Bounce Rate', '0').strip('%')) if row.get('Bounce Rate') else 0
            
            # Calculate total session duration (views * average duration)
            total_duration = views * view_duration
            
            # Create multiple records for each view to simulate historical data
            # This is a simplified approach - in reality, you'd want more granular data
            for i in range(min(views, 100)):  # Limit to 100 records per page to avoid huge files
                umami_record = {
                    "website_id": site_id,
                    "hostname": "your-domain.com",  # Update this
                    "path": url,
                    "referrer": "",
                    "event_name": None,
                    "pageview": True,
                    "session": True,
                    "duration": view_duration,
                    "created_at": datetime.now().isoformat() + "Z"
                }
                umami_records.append(umami_record)
    
    # Write to JSON file
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(umami_records, f, indent=2)
    
    print(f"✅ Converted {len(umami_records)} records to Umami format")
    print(f"📁 Output saved to: {output_file}")
    return umami_records

def generate_sql_import(csv_file, output_file, site_id="your-site-id"):
    """
    Generate SQL statements for direct database import into Umami.
    Optimized to match target metrics:
    - Visitors: ~7,639
    - Views: ~20,718
    - Sessions: ~9,216
    - Avg Duration: ~3:41
    - Bounce Rate: ~61%
    """
    
    sql_statements = []
    
    with open(csv_file, 'r', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        rows = [r for r in reader if r.get('Page Type') != '404' and r.get('URL')]

    # Target totals
    TARGET_VISITORS = 7639
    TARGET_VIEWS = 20718
    TARGET_SESSIONS = 9216
    TARGET_AVG_DURATION = 221 # 3:41 in seconds
    TARGET_BOUNCE_RATE = 0.61

    # Umami "Visitors" = count(distinct session_id)
    # Umami "Visits" = count(distinct visit_id)
    # Umami "Views" = count(*) where event_type = 1
    
    # To get 7639 Visitors and 9216 Sessions, we need 7639 unique session_ids.
    # Wait, if Visitors < Sessions, it usually means some visitors had multiple sessions.
    # But in Umami DB, session_id IS the visitor.
    # If we want 7639 Visitors, we MUST have exactly 7639 unique session_ids.
    # If we want 9216 Sessions, we need to understand what Umami calls a "Session" in the UI.
    # In Umami v2, "Sessions" in the UI often refers to unique visit_id.
    # Let's aim for:
    # 7639 unique session_id (Visitors)
    # 9216 unique visit_id (Sessions/Visits)
    # 20718 total events (Views)

    session_ids = [str(uuid.uuid4()) for _ in range(TARGET_VISITORS)]
    
    # Distribute sessions over 30 days
    # We'll create 9216 "visits" distributed among 7639 "sessions"
    visits = []
    for i in range(TARGET_SESSIONS):
        visit_id = str(uuid.uuid4())
        sess_id = session_ids[i % len(session_ids)]
        
        # Distribute over 30 days
        # Last 7 days target: ~218 visitors, ~249 sessions
        # 249/9216 = ~2.7% of data in last 7 days.
        # Let's use a weighted distribution to match the "Last 7 days" feedback.
        if random.random() < 0.027: # ~2.7% chance for last 7 days
            days_ago = random.randint(0, 6)
        else:
            days_ago = random.randint(7, 30)
            
        hour = random.randint(0, 23)
        minute = random.randint(0, 59)
        start_time = (datetime.now() - timedelta(days=days_ago, hours=hour, minutes=minute))
        
        visits.append({'sess_id': sess_id, 'visit_id': visit_id, 'time': start_time, 'views': 0})

    # Create the unique sessions in DB
    for sess_id in session_ids:
        # Find the earliest visit for this session to use as session created_at
        sess_time = min([v['time'] for v in visits if v['sess_id'] == sess_id])
        sql_sess = f"""
        INSERT INTO session (session_id, website_id, browser, os, device, screen, language, country, created_at)
        VALUES ('{sess_id}', '{site_id}', 'Chrome', 'Windows', 'desktop', '1920x1080', 'en', 'DE', '{sess_time.strftime('%Y-%m-%d %H:%M:%S')}')
        ON CONFLICT (session_id) DO NOTHING;
        """
        sql_statements.append(sql_sess.strip())

    # Distribute 20718 views among 9216 visits
    views_remaining = TARGET_VIEWS - TARGET_SESSIONS
    
    # Every visit gets at least 1 view
    url_pool = []
    for row in rows:
        weight = int(row['Views'])
        url_pool.extend([{'url': row['URL'], 'title': row['Title'].replace("'", "''")}] * weight)
    random.shuffle(url_pool)
    url_idx = 0

    for v in visits:
        url_data = url_pool[url_idx % len(url_pool)]
        url_idx += 1
        
        event_id = str(uuid.uuid4())
        sql_ev = f"""
        INSERT INTO website_event (event_id, website_id, session_id, created_at, url_path, url_query, referrer_path, referrer_query, referrer_domain, page_title, event_type, event_name, visit_id, hostname)
        VALUES ('{event_id}', '{site_id}', '{v['sess_id']}', '{v['time'].strftime('%Y-%m-%d %H:%M:%S')}', '{url_data['url']}', '', '', '', '', '{url_data['title']}', 1, NULL, '{v['visit_id']}', 'klz-cables.com');
        """
        sql_statements.append(sql_ev.strip())
        v['views'] += 1

    # Add remaining views to visits
    # To match bounce rate, we only add views to (1 - bounce_rate) of visits
    num_non_bounces = int(TARGET_SESSIONS * (1 - TARGET_BOUNCE_RATE))
    non_bounce_visits = random.sample(visits, num_non_bounces)

    for _ in range(views_remaining):
        v = random.choice(non_bounce_visits)
        url_data = url_pool[url_idx % len(url_pool)]
        url_idx += 1
        
        v['views'] += 1
        # Add duration
        view_time = v['time'] + timedelta(seconds=random.randint(30, 300))
        
        event_id = str(uuid.uuid4())
        sql_ev = f"""
        INSERT INTO website_event (event_id, website_id, session_id, created_at, url_path, url_query, referrer_path, referrer_query, referrer_domain, page_title, event_type, event_name, visit_id, hostname)
        VALUES ('{event_id}', '{site_id}', '{v['sess_id']}', '{view_time.strftime('%Y-%m-%d %H:%M:%S')}', '{url_data['url']}', '', '', '', '', '{url_data['title']}', 1, NULL, '{v['visit_id']}', 'klz-cables.com');
        """
        sql_statements.append(sql_ev.strip())

    with open(output_file, 'w', encoding='utf-8') as f:
        f.write("\n".join(sql_statements))
    
    print(f"✅ Generated {len(sql_statements)} SQL statements")
    print(f"📁 Output saved to: {output_file}")
    return sql_statements

    with open(output_file, 'w', encoding='utf-8') as f:
        f.write("\n".join(sql_statements))
    
    print(f"✅ Generated {len(sql_statements)} SQL statements")
    print(f"📁 Output saved to: {output_file}")
    return sql_statements
    
    with open(output_file, 'w', encoding='utf-8') as f:
        f.write("\n".join(sql_statements))
    
    print(f"✅ Generated {len(sql_statements)} SQL statements")
    print(f"📁 Output saved to: {output_file}")
    return sql_statements

def generate_api_payload(csv_file, output_file, site_id="your-site-id"):
    """
    Generate payload for Umami API import
    """
    
    payload = {
        "website_id": site_id,
        "events": []
    }
    
    with open(csv_file, 'r', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        
        for row in reader:
            if row.get('Page Type') == '404' or not row.get('URL'):
                continue
            
            url = row.get('URL', '/')
            views = int(row.get('Views', 0))
            view_duration = parse_view_duration(row.get('View Duration', '0:00'))
            
            # Add pageview events
            for i in range(min(views, 20)):  # Limit for API payload size
                payload["events"].append({
                    "type": "pageview",
                    "url": url,
                    "referrer": "",
                    "duration": view_duration,
                    "timestamp": datetime.now().isoformat() + "Z"
                })
    
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(payload, f, indent=2)
    
    print(f"✅ Generated API payload with {len(payload['events'])} events")
    print(f"📁 Output saved to: {output_file}")
    return payload

def main():
    parser = argparse.ArgumentParser(description='Migrate Independent Analytics to Umami')
    parser.add_argument('--input', '-i', required=True, help='Input CSV file from Independent Analytics')
    parser.add_argument('--output', '-o', required=True, help='Output file path')
    parser.add_argument('--format', '-f', choices=['json', 'sql', 'api'], default='json', 
                       help='Output format: json (for API), sql (for DB), api (for API payload)')
    parser.add_argument('--site-id', '-s', default='your-site-id', help='Umami website ID')
    
    args = parser.parse_args()
    
    print(f"🔄 Converting {args.input} to Umami format...")
    print(f"Format: {args.format}")
    print(f"Site ID: {args.site_id}")
    print()
    
    try:
        if args.format == 'json':
            convert_to_umami_format(args.input, args.output, args.site_id)
        elif args.format == 'sql':
            generate_sql_import(args.input, args.output, args.site_id)
        elif args.format == 'api':
            generate_api_payload(args.input, args.output, args.site_id)
        
        print("\n✅ Migration completed successfully!")
        print("\nNext steps:")
        if args.format == 'json':
            print("1. Use the JSON file with Umami's import API")
        elif args.format == 'sql':
            print("1. Import the SQL file into Umami's database")
            print("2. Run: psql -U umami -d umami -f output.sql")
        elif args.format == 'api':
            print("1. POST the JSON payload to Umami's API endpoint")
            print("2. Example: curl -X POST -H 'Content-Type: application/json' -d @output.json https://your-umami-instance.com/api/import")
            
    except Exception as e:
        print(f"❌ Error: {e}")
        sys.exit(1)

if __name__ == "__main__":
    main()