Sync your data

#!/usr/bin/env python

import csv
import json
import collections
import requests as requests

OrderedDict = collections.OrderedDict

SOURCE = "imdb"
DATA_SYNC_ENDPOINT = "https://cg.optimizely.com/api/content/v2/data?id={}".format(SOURCE)
HEADERS = {
    'Content-Type': 'text/plain',
    'Authorization': 'Basic <Token>'
}

NAME_BASICS_FILE = 'data/name.basics.small.tsv'
TITLE_BASICS_FILE = 'data/title.basics.small.tsv'
TITLE_RATINGS_FILE = 'data/title.ratings.small.tsv'

STRING_ARRAY_FIELDS = ["ContentType", "knownForTitles", "primaryProfession___searchable", "genres___searchable"]
INT_FIELDS = ["birthYear", "deathYear", "startYear", "endYear", "runtimeMinutes", "numVotes"]
FLOAT_FIELDS = ["averageRating"]
BOOLEAN_FIELDS = ["isAdult"]

def load_data(source, content_type):
    data = []
    with open(source, 'r') as csvfile:
        reader = csv.reader(csvfile, delimiter='\t', quotechar='"')
        header = next(reader)
        header.append("ContentType")
        header.append("Status")
        header.append("_rbac")
        header.append("__typename")
        count = 0
        idx = 0
        for (is_last_check, row) in is_last(reader):
            for i, value in enumerate(row):
                if header[i] in STRING_ARRAY_FIELDS:
                    row[i] = value.split(",") if "," in value else [value]
                elif header[i] in INT_FIELDS:
                    row[i] = int(value) if value != "\\N" else None
                elif header[i] in FLOAT_FIELDS:
                    row[i] = float(value) if value != "\\N" else None
                elif header[i] in BOOLEAN_FIELDS:
                    row[i] = value.lower() in ["1"]
            row.append(["Record", content_type])
            row.append("Published")
            row.append("r:Everyone:Read")
            row.append(content_type)
            data.append(OrderedDict(zip(header, row)))

            count += 1
            idx += 1
            if count == 100 or is_last_check:
                count = 0
                bulk = '\n'.join(
                    "{\"index\": { \"_id\": \"" + source + str(idx + i) + "\", \"language_routing\": \"en\" }}\n" + json.dumps(v) for (i, v) in
                    enumerate(data))
                response = requests.request("POST", DATA_SYNC_ENDPOINT, headers=HEADERS, data=bulk)
                print(response.text)
                data = []


def is_last(itr):
    old = next(itr)
    for new in itr:
        yield False, old
        old = new
    yield True, old


load_data(NAME_BASICS_FILE, "Actor")
load_data(TITLE_BASICS_FILE, "Title")
load_data(TITLE_RATINGS_FILE, "Rating")