configs.yaml

general_configs:
    verbose: &verbose True
    raw_data: &raw_data data/raw/
    interim_data: &interim_data data/interim/
    processed_data: &processed_data data/processed/
    external_data: &external_data data/external/
    elasticsearch_ip: &es_ip localhost

twitter_data_pull:
    credentials_path: twitter_creds.yaml
    credentials_key: search_tweets_api
    save_path: *raw_data
    results_per_call: 500
    max_results: 450000
    file_name: full_data_pull.json
    from_date: "2020-05-01"
    to_date: "2020-06-07"
    query: Amphan OR Ampan OR अम्फान OR ঘূর্ণিঝড় OR হ্যারিকেন OR আম্পান OR আমপান OR অ্যাম্ফান OR ସାଇକ୍ଲୋନ୍ OR ମହାବାତ୍ୟାର OR ଆମ୍ଫାନ୍ -nisarga -(ARB 01) -নিসর্গ -ब्रह्मांड -ବ୍ରହ୍ମାଣ୍ଡ
    count_bucket: "day"
    verbose: *verbose

test_data_pull:
    credentials_path: twitter_creds.yaml
    credentials_key: sample_tweets_api
    save_path: *raw_data
    results_per_call: 100
    max_results: 300
    from_date: "2020-05-14"
    to_date: "2020-06-15"
    query: ((cyclone OR cyclonic OR cyclonical OR hurricane OR storm) (Amphan OR Ampan))
    count_bucket: "day"
    verbose: *verbose

data_etl:
    input_path: data/raw/chunked_data/
    # ip_address: *es_ip
    load_es: False
    # ignore save_path and uncomment ip_address if load_es is True
    save_path: *processed_data
    verbose: *verbose

load_es:
    input_path: *processed_data 
    ip_address: *es_ip
    verbose: *verbose

update_es:
    ip_address: *es_ip
    input_path: *processed_data 
    verbose: *verbose