handle themes that come from gitlab (instead of from github),

i.e. Midnight https://gitlab.com/BluestNight/Midnight

rank_hugo_themes.py

@@ -13,8 +13,9 @@ from base64 import b64decode
 from ast import literal_eval
 
 DATABASENAME='hugothemes.db'
-# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int);
+# CREATE TABLE hugothemes (name varchar unique, ETag text, url text, jsondump text, commit_sha text, commit_date text, commit_date_in_seconds int, repo_ETag text, stargazers_count int, themes_toml_ETag text, themes_toml_content text, tags_list text, num_tags int, dot_gitmodules_content text);
 # CREATE TABLE tags (tag varchar unique, theme_list text, num_themes int);
+# CREATE TABLE hugothemes_from_gitlab (name varchar unique, url text, commit_sha text, id text, commit_date_in_seconds int, commit_date text, star_count int, themes_toml_content text);
 THEMESLISTREPO='gohugoio/hugoThemes'
 
 if len(sys_argv) == 2:
@@ -57,9 +58,68 @@ def get_hugo_themes_list():
     dbconnection.close()
 
 
+def update_hugo_themes_submodule_list():
+    dbconnection = connect(DATABASENAME)
+    cursor = dbconnection.cursor()
+    cursor.execute("select jsondump,dot_gitmodules_content from hugothemes where name=?",(THEMESLISTREPO,))
+    themes_json_string,dot_gitmodule_json_string = cursor.fetchone()
+    themes_json = json_loads(themes_json_string)
+    if dot_gitmodule_json_string:
+        dot_gitmodule_json = json_loads(dot_gitmodule_json_string)
+    for theme in themes_json:
+        if theme['name'] == '.gitmodules':
+            new_gitmodules_sha = theme['sha']
+            break
+    if not dot_gitmodule_json_string or new_gitmodules_sha != dot_gitmodule_json['sha']:
+        api_call_url = "https://api.github.com/repos/"+THEMESLISTREPO+"/contents/.gitmodules"
+        response = get(api_call_url)
+        if response.status_code == 200:
+            sql = "update hugothemes set dot_gitmodules_content=? where name=?"
+            values=(response.text,THEMESLISTREPO)
+            cursor.execute(sql,values)
+            dbconnection.commit()
+        print(response.status_code)
+    cursor.close()
+    dbconnection.close()
+
+
+def get_hugo_themes_submodule_list():
+    dbconnection = connect(DATABASENAME)
+    cursor = dbconnection.cursor()
+    cursor.execute("select dot_gitmodules_content from hugothemes where name=?",(THEMESLISTREPO,))
+    dot_gitmodule_json_string = cursor.fetchone()[0]
+    dot_gitmodule_json = json_loads(dot_gitmodule_json_string)
+    dot_gitmodule_content = b64decode(dot_gitmodule_json['content']).decode('utf-8').replace('\n\n','\n').split('\n')
+    submodules = []
+    for line in range(len(dot_gitmodule_content[:-1])):
+        if 'submodule' in dot_gitmodule_content[line]:
+            name = re.sub(r'(^\[submodule "|"\]$)','',dot_gitmodule_content[line])
+            if 'path = ' in dot_gitmodule_content[line+1]:
+                path = re.match(r'^.*(path = )(.*)$',dot_gitmodule_content[line+1]).group(2)
+            if 'url = ' in dot_gitmodule_content[line+2]:
+                url = re.match(r'^.*(url = )(.*)$',dot_gitmodule_content[line+2]).group(2)
+            submodules.append((name,path,url))
+    cursor.close()
+    dbconnection.close()
+    return submodules
+
+
 def clean_up():
     dbconnection = connect(DATABASENAME)
     cursor = dbconnection.cursor()
+    new_name_list = []
+    submodules = get_hugo_themes_submodule_list()
+    for submodule in submodules:
+        submodule_name = re.sub(r'(^https://github.com/|^https://gitlab.com/|\.git$)','',submodule[2])
+        new_name_list.append(submodule_name)
+    cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,))
+    old_name_list = [name[0] for name in cursor]
+
+    for name in old_name_list:
+        if name not in new_name_list:
+            cursor.execute("delete from hugothemes where name=?",(name,))
+            dbconnection.commit()
+
     cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,))
     themes_json_string = cursor.fetchone()[0]
     themes_json = json_loads(themes_json_string)
@@ -70,16 +130,62 @@ def clean_up():
                 split_html_url = theme['html_url'].split('/')
                 new_short_name = split_html_url[3]+'/'+split_html_url[4]
                 new_name_list.append(new_short_name)
-    cursor.execute("select name from hugothemes where name!=?",(THEMESLISTREPO,))
+
-    old_name_list = cursor.fetchall()
     for name in old_name_list:
-        if name[0] not in new_name_list:
+        if name not in new_name_list:
-            cursor.execute("delete from hugothemes where name=?",(name[0],))
+            cursor.execute("delete from hugothemes where name=?",(name,))
             dbconnection.commit()
+
     cursor.close()
     dbconnection.close()
 
 
+def get_non_github_themes():
+    dbconnection = connect(DATABASENAME)
+    cursor = dbconnection.cursor()
+    cursor.execute("select name from hugothemes_from_gitlab;")
+    non_github_theme_list = [theme[0] for theme in cursor]
+    cursor.close()
+    dbconnection.close()
+    return non_github_theme_list
+
+
+def parse_submodules_from_gitlab():
+    submodules = get_hugo_themes_submodule_list()
+    temp_submodules_list = []
+    for submodule in submodules:
+        if 'gitlab' in submodule[2]:
+            temp_submodules_list.append(submodule)
+    if (len(temp_submodules_list) > 0):
+        dbconnection = connect(DATABASENAME)
+        cursor = dbconnection.cursor()
+        cursor.execute("select jsondump from hugothemes where name=?",(THEMESLISTREPO,))
+        themes_json_string = cursor.fetchone()[0]
+        themes_json = json_loads(themes_json_string)
+        cursor.close()
+        dbconnection.close()
+        submodules_list = []
+        for submodule in temp_submodules_list:
+            for theme in themes_json:
+                if submodule[0] == theme['name']:
+                    if (theme['html_url'] == None) or (theme['git_url'] == None):
+                        submodules_list.append((submodule[0],submodule[1],submodule[2],theme['sha']))
+        if (len(submodules_list) > 0):
+            dbconnection = connect(DATABASENAME)
+            cursor = dbconnection.cursor()
+            for submodule in submodules_list:
+                url = re.sub(r'\.git$','',submodule[2])
+                theme_name = url[19:]
+                cursor.execute("insert or ignore into hugothemes_from_gitlab (name) values (?)",[theme_name])
+                dbconnection.commit()
+                sql = "update hugothemes_from_gitlab set url=?,commit_sha=? where name=?"
+                values=(url,submodule[3],theme_name)
+                cursor.execute(sql,values)
+                dbconnection.commit()
+            cursor.close()
+            dbconnection.close()
+
+
 def parse_hugo_themes_list():
     dbconnection = connect(DATABASENAME)
     cursor = dbconnection.cursor()
@@ -101,12 +207,34 @@ def parse_hugo_themes_list():
     dbconnection.close()
 
 
+def get_gitlab_project_ids():
+    dbconnection = connect(DATABASENAME)
+    cursor = dbconnection.cursor()
+    cursor.execute("select name,url,id from hugothemes_from_gitlab;")
+    themes = [row for row in cursor]
+    for theme in themes:
+        if theme[2] == None:
+            response = get(theme[1])
+            id_result = re.search(r'(Project ID: )(\d{5,})',response.text).group(2)
+            sql = "update hugothemes_from_gitlab set id=? where name=?"
+            values=(id_result,theme[0])
+            cursor.execute(sql,values)
+            dbconnection.commit()
+    cursor.close()
+    dbconnection.close()
+    return True
+
+
 def get_commit_info_for_hugo_themes():
+    non_github_theme_list = get_non_github_themes()
+    themes = []
     dbconnection = connect(DATABASENAME)
     cursor = dbconnection.cursor()
     cursor.execute("select name,ETag,url,commit_sha from hugothemes where name!=?",(THEMESLISTREPO,))
-    hugo_themes_list = cursor.fetchall()
+    for theme in cursor:
-    for theme in hugo_themes_list:
+        if theme[0] not in non_github_theme_list:
+            themes.append(theme)
+    for theme in themes:
         name,old_ETag,url,commit_sha = theme
         api_call_url = 'https://api.github.com/repos/' + name + '/commits/' + commit_sha
 
@@ -140,12 +268,46 @@ def get_commit_info_for_hugo_themes():
     dbconnection.close()
     
 
+def get_commit_info_for_hugo_themes_from_gitlab():
+    if get_gitlab_project_ids():
+        dbconnection = connect(DATABASENAME)
+        cursor = dbconnection.cursor()
+        cursor.execute("select name,url,commit_sha,id from hugothemes_from_gitlab")
+        hugo_themes_list = cursor.fetchall()
+        for theme in hugo_themes_list:
+            name,url,commit_sha,gitlab_project_id = theme
+            api_call_url = 'https://gitlab.com/api/v4/projects/' + gitlab_project_id + '/repository/commits/' + commit_sha
+
+            response = get(api_call_url)
+            if response.status_code == 200:
+                result = response.json()
+                commit_date = re.sub(r'(\.\d{3})(Z$)',r'\2',result['created_at'])
+                commit_date_in_seconds = timegm(strptime(commit_date,'%Y-%m-%dT%H:%M:%SZ'))
+                sql = "update hugothemes_from_gitlab set commit_date=?,commit_date_in_seconds=? where name=?"
+                values=(commit_date,commit_date_in_seconds,name)
+                cursor.execute(sql,values)
+                dbconnection.commit()
+            elif response.status_code == 403:
+                print(response.status_code)
+                cursor.close()
+                dbconnection.close()
+                write_reports()
+                quit()
+            print(response.status_code)
+        cursor.close()
+        dbconnection.close()
+    
+
 def get_stargazer_count_for_hugo_themes():
+    non_github_theme_list = get_non_github_themes()
+    themes = []
     dbconnection = connect(DATABASENAME)
     cursor = dbconnection.cursor()
     cursor.execute("select name,repo_ETag from hugothemes where name!=?",(THEMESLISTREPO,))
-    hugo_themes_list = cursor.fetchall()
+    for theme in cursor:
-    for theme in hugo_themes_list:
+        if theme[0] not in non_github_theme_list:
+            themes.append(theme)
+    for theme in themes:
         name,old_ETag = theme
         api_call_url = 'https://api.github.com/repos/' + name
 
@@ -178,12 +340,44 @@ def get_stargazer_count_for_hugo_themes():
     dbconnection.close()
 
 
+def get_stargazer_count_for_hugo_themes_from_gitlab():
+    if get_gitlab_project_ids():
+        dbconnection = connect(DATABASENAME)
+        cursor = dbconnection.cursor()
+        cursor.execute("select name,id from hugothemes_from_gitlab;")
+        themes = cursor.fetchall()
+        for theme in themes:
+            api_call_url = 'https://gitlab.com/api/v4/projects/' + theme[1]
+
+            response = get(api_call_url)
+            if response.status_code == 200:
+                result = response.json()
+                star_count = result['star_count']
+                sql = "update hugothemes_from_gitlab set star_count=? where name=?"
+                values=(star_count,theme[0])
+                cursor.execute(sql,values)
+                dbconnection.commit()
+            elif response.status_code == 403:
+                print(response.status_code)
+                cursor.close()
+                dbconnection.close()
+                write_reports()
+                quit()
+            print(response.status_code)
+        cursor.close()
+        dbconnection.close()
+
+
 def get_theme_dot_toml_for_each_hugo_themes():
+    non_github_theme_list = get_non_github_themes()
+    themes = []
     dbconnection = connect(DATABASENAME)
     cursor = dbconnection.cursor()
     cursor.execute("select name,themes_toml_ETag from hugothemes where name!=?",(THEMESLISTREPO,))
-    hugo_themes_list = cursor.fetchall()
+    for theme in cursor:
-    for theme in hugo_themes_list:
+        if theme[0] not in non_github_theme_list:
+            themes.append(theme)
+    for theme in themes:
         name,old_ETag = theme
         api_call_url = "https://api.github.com/repos/"+name+"/contents/theme.toml"
 
@@ -216,6 +410,51 @@ def get_theme_dot_toml_for_each_hugo_themes():
     dbconnection.close()
 
 
+def get_theme_dot_toml_for_each_hugo_themes_from_gitlab():
+    if get_gitlab_project_ids():
+        dbconnection = connect(DATABASENAME)
+        cursor = dbconnection.cursor()
+        cursor.execute("select name,id from hugothemes_from_gitlab")
+        themes = cursor.fetchall()
+        for theme in themes:
+            api_call_url = 'https://gitlab.com/api/v4/projects/' + theme[1] + '/repository/files/theme.toml?ref=master'
+
+            response = get(api_call_url)
+            if response.status_code == 200:
+                result = response.json()
+                themes_toml_content = result['content']
+                sql = "update hugothemes_from_gitlab set themes_toml_content=? where name=?"
+                values=(themes_toml_content,theme[0])
+                cursor.execute(sql,values)
+                dbconnection.commit()
+            elif response.status_code == 403:
+                print(response.status_code)
+                cursor.close()
+                dbconnection.close()
+                write_reports()
+                quit()
+            print(response.status_code)
+        cursor.close()
+        dbconnection.close()
+
+
+def coalesce_themes():
+    if get_gitlab_project_ids():
+        dbconnection = connect(DATABASENAME)
+        cursor = dbconnection.cursor()
+        cursor.execute("select name,url,commit_sha,commit_date,commit_date_in_seconds,star_count,themes_toml_content from hugothemes_from_gitlab")
+        themes = cursor.fetchall()
+        for theme in themes:
+            cursor.execute("insert or ignore into hugothemes (name) values (?)",[theme[0]])
+            dbconnection.commit()
+            sql = "update hugothemes set url=?,commit_sha=?,commit_date=?,commit_date_in_seconds=?,stargazers_count=?,themes_toml_content=? where name=?"
+            values=(theme[1],theme[2],theme[3],theme[4],theme[5],theme[6],theme[0])
+            cursor.execute(sql,values)
+            dbconnection.commit()
+        cursor.close()
+        dbconnection.close()
+
+
 def update_tags_list_for_each_hugo_themes():
     dbconnection = connect(DATABASENAME)
     cursor = dbconnection.cursor()
@@ -302,7 +541,6 @@ def make_nav_buttons(button_info):
     return button_block
 
 
-
 def make_table(themes_info):
     table = "\t<div class='container'>\n\t\t<table class='table monospace'>\n\t\t\t<thead><tr><th scope='col'>" + themes_info[1] + " (tag) " + themes_info[2] + "</th><th scope='col'>Date</th></tr></thead>\n\t\t\t<tbody>\n"
     for theme in themes_info[0]:
@@ -367,12 +605,19 @@ def write_reports():
 
 if __name__=="__main__":
     get_hugo_themes_list()
+    update_hugo_themes_submodule_list()
     clean_up()
+    parse_submodules_from_gitlab()
     parse_hugo_themes_list()
     get_commit_info_for_hugo_themes()
+    get_commit_info_for_hugo_themes_from_gitlab()
     get_stargazer_count_for_hugo_themes()
+    get_stargazer_count_for_hugo_themes_from_gitlab()
     get_theme_dot_toml_for_each_hugo_themes()
+    get_theme_dot_toml_for_each_hugo_themes_from_gitlab()
+    coalesce_themes()
     update_tags_list_for_each_hugo_themes()
+    update_tag_table()
     write_reports()
     """
     """