git.lirion.de

Of git, get, and gud

aboutsummaryrefslogtreecommitdiffstats
path: root/refresh
blob: f6b5cc486c53563026020a27d9f0a4b6b596d4a2 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim:syntax=python

"""
This script downloads data from holidata.net and writes it into files in the current directory.
Its output formats are parseable by timewarrior (https://taskwarrior.org/docs/timewarrior/).

This is very alpha. Since holidata.net has a few minor mistakes for Germany I'm writing this to
    test.xx-YY for now (I've just reported these, so allow the guys some time :) ). The files
    are still sourcable with that name :)

Also, for now this script only downloads de-DE. You will notice the complete lack of a main()
    definition, that is the next step: it will also carry parameters so you can download your
    own stuff.

Finally, I'm fixing the ISO-3166-2 violation for North Rhine Westphalia - holidata.net has de-NRW.
    That is the next bug to fix for them :)

tl;dr
SCRIPT IS NOT FINISHED, THIS IS A GIT - USE ONLY IF YOU UNDERSTAND AND IF YOU ARE FROM ZE GERMANY
"""

#import os, sys, re, time, argparse
import json, sys, argparse
reload(sys)
sys.setdefaultencoding('utf8')
from datetime import datetime
from time import sleep

if sys.version_info >= (3, 0):
    from urllib.request import urlopen
    from urllib.error import HTTPError
else:
    from urllib2 import urlopen, HTTPError

__hheader__ = "# File auto-generated by lirion.de's refresh,\n" \
            "# Date: " + datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S %z") + "\n" \
            "# data provided by holidata.net\n\n" \
            "define holidays:\n"

def headwrite(file):
    """
    This writes the default header to the file it is being passed on. Content see
    __hheader__ right above.
    """
    try:   
        open(file, 'w').close()
    except IOError:
        print("Cannot write to %s!", file)
        exit(2)
    with open(file, "a") as ofile:
        #f = open(ofile, "a")
        ofile.write(__hheader__)
 
def hfetch(locode, year):
    """
    This function downloads the holiday data for a given ISO-3166 code from holidata.net
    It will return a single string containing all of the JSON.
    """
    lines = ""
    myurl = "https://holidata.net/" + locode + "/" + year + ".json"
    try:
        lines = urlopen(myurl).read().decode('utf-8')
    except HTTPError as httpe:
        if httpe.code == 404:
            print("holidata.net does not have data for %s, for %s." % (locode, year))
        else:
            print(httpe.code, httpe.read())
    return lines

def hparse(lines, locode, year):
    """
    This function parses the JSON string we expect from holidata.net and writes its fragments to
    the specific region files, if a region is given. So, e.g.: a holiday for all of the US will
    not have a region definition and thus be written to file.en-US, whereas a holiday that is not
    valid for all of the US but is valid for Oregon will be written to file.us-OR.
    """
    islocdef = {}
    for line in lines.split('\n'):
        if line:
            sys.stdout.write(".")
            jdata = json.loads(line)
            mylocale = unicode(jdata['locale'])
            mycountry = mylocale.split('-')[1].lower()
            myregion = unicode(jdata['region'])
            if myregion == "":
                myregion = mylocale.split('-')[1]
            # holidata.net violates ISO-3166-2:DE here:
            if myregion == "NRW":
                myregion = "NW" # or Neustadt/Weinstr., Palz > Pott
            myhlocale = unicode(mycountry + "-" + myregion)
            if jdata['date'] == "":
                print("%s has no date given for %s!", jdata['description'], jdata['locale'])
                mydate = unicode('1970_01_01')
            else:
                mydate = unicode(datetime.strftime(datetime.strptime(jdata['date'], '%Y-%m-%d'),
                    '%Y_%m_%d'))
            myfile = "test." + myhlocale
            if myhlocale not in islocdef:
                headwrite(myfile)
                with open(myfile, "a") as ofile:
                    #f = open(ofile, "a")
                    ofile.write("  " + mycountry + "-" + myregion + ":\n")
                islocdef[myhlocale] = myhlocale
            with open(myfile, "a") as ofile:
                ofile.write("    " + mydate + " = " + unicode(jdata['description']) + "\n")
            sleep(0.1)
            sys.stdout.flush()

def main(args):
    if args.locale != [] and args.locale != None:
        locode = str(args.locale[0])
    else:
        locode = ["de-DE"]
    if args.year != None and args.year != []:
        year = args.year
    else:
        now = datetime.now()
        year = [datetime.now().year]
    year = unicode(year)
    for i in args.locale:
        for j in args.year:
            sys.stdout.write("Fetching holiday data from holidata.net... (%s, %s)" % (i, j))
            sys.stdout.flush()
            lines = hfetch(unicode(i), unicode(j))
            print(" done.")
            if lines == "":
                print("No lines returned from holidata.net for %s!", locode)
                exit(3)
            sys.stdout.write("Parsing data")
            sys.stdout.flush()
            hparse(lines, unicode(i), unicode(j))
            print(" done.")

if __name__ == "__main__":
    usage = """See https://holidata.net for details of supported locales and regions."""
    parser = argparse.ArgumentParser(description="Update holiday data files. Run 'refresh' for \
        the default of de-DE (this will be changed in the future)")
    parser.add_argument('--locale', nargs='+', help='Specific locale to update', type=unicode, default=[])
    parser.add_argument('--region', nargs='+', help='Specific locale region to update', type=unicode, default=[])
    parser.add_argument('--year',   nargs='+', help='Specific year to fetch.', type=int, default=[])
    args = parser.parse_args()
    main(args)
    #try:
    #    main(args)
    #except Exception as msg:
    #    print('Error:',msg)