diff --git a/rank_hugo_themes.py b/rank_hugo_themes.py index ad165e3..394e3ad 100755 --- a/rank_hugo_themes.py +++ b/rank_hugo_themes.py @@ -94,7 +94,8 @@ else: def get_hugo_themes_list(): - themes_list_url = f"https://raw.githubusercontent.com/{THEMESLISTREPO}/main/themes.txt" + themes_list_url = "https://raw.githubusercontent.com/" + themes_list_url += f"{THEMESLISTREPO}/main/themes.txt" response = get(themes_list_url) if response.status_code == 200: @@ -111,35 +112,52 @@ def get_hugo_themes_list(): def clean_up(): themes_name_list = get_themes_name_list() session = sessionmaker(bind=engine)() - hugo_themes_name_list = [theme[0] for theme in session.query(Hugothemes.name).all()] + hugo_themes_name_list = [ + theme[0] for theme in session.query(Hugothemes.name).all()] for theme_name in hugo_themes_name_list: if theme_name not in themes_name_list: - removed_theme = session.query(Hugothemes).filter_by(name=theme_name).first() + removed_theme = session.query( + Hugothemes).filter_by(name=theme_name).first() session.delete(removed_theme) session.commit() gitlab_themes_name_list = get_gitlab_themes_name_list() - hugo_themes_from_gitlab_name_list = [theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] + hugo_themes_from_gitlab_name_list = [ + theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()] for theme in hugo_themes_from_gitlab_name_list: if theme not in gitlab_themes_name_list: - removed_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).first() + removed_theme = session.query( + Hugothemes_from_gitlab).filter_by(name=theme).first() session.delete(removed_theme) session.commit() def dedup_database(): session = sessionmaker(bind=engine)() - hugo_themes_sha_list = [theme[0] for theme in session.query(Hugothemes.commit_sha).all()] - sha_list = [sha for sha in hugo_themes_sha_list if hugo_themes_sha_list.count(sha) > 1] + hugo_themes_sha_list = [ + theme[0] for theme in session.query(Hugothemes.commit_sha).all()] + sha_list = [ + sha for sha in hugo_themes_sha_list if hugo_themes_sha_list.count( + sha + ) > 1 + ] for sha in sha_list: - removed_theme = session.query(Hugothemes).filter_by(commit_sha=sha).first() + removed_theme = session.query( + Hugothemes).filter_by(commit_sha=sha).first() session.delete(removed_theme) session.commit() - hugo_themes_sha_list_from_gitlab = [theme[0] for theme in session.query(Hugothemes_from_gitlab.commit_sha).all()] - sha_list_from_gitlab = [sha for sha in hugo_themes_sha_list_from_gitlab if hugo_themes_sha_list_from_gitlab.count(sha) > 1] + hugo_themes_sha_list_from_gitlab = [theme[0] for theme in session.query( + Hugothemes_from_gitlab.commit_sha).all()] + sha_list_from_gitlab = [ + sha for sha in hugo_themes_sha_list_from_gitlab + if hugo_themes_sha_list_from_gitlab.count( + sha + ) > 1 + ] for sha in sha_list_from_gitlab: - removed_theme = session.query(Hugothemes_from_gitlab).filter_by(commit_sha=sha).first() + removed_theme = session.query( + Hugothemes_from_gitlab).filter_by(commit_sha=sha).first() session.delete(removed_theme) session.commit() @@ -149,7 +167,8 @@ def parse_gitlab_hugo_themes_list(): gitlab_themes_list = get_gitlab_themes_list() for theme in gitlab_themes_list: theme_name = theme[11:] - existing_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme_name).first() + existing_theme = session.query( + Hugothemes_from_gitlab).filter_by(name=theme_name).first() if existing_theme is None: session.add(Hugothemes_from_gitlab(name=theme_name, url=theme)) session.commit() @@ -163,7 +182,8 @@ def parse_hugo_themes_list(): session = sessionmaker(bind=engine)() for theme in THEMESLIST: theme_name = theme[11:] - existing_theme = session.query(Hugothemes).filter_by(name=theme_name).first() + existing_theme = session.query( + Hugothemes).filter_by(name=theme_name).first() if existing_theme is None: session.add(Hugothemes(name=theme_name, url=theme)) session.commit() @@ -175,17 +195,21 @@ def parse_hugo_themes_list(): def get_gitlab_project_ids(): session = sessionmaker(bind=engine)() - themes = (theme[0] for theme in session.query(Hugothemes_from_gitlab.name).all()) + themes = (theme[0] for theme in session.query( + Hugothemes_from_gitlab.name).all()) match = re.compile(r'(Project ID: )(\d{5,})') for theme in themes: - gitlab_theme = session.query(Hugothemes_from_gitlab).filter_by(name=theme).one() + gitlab_theme = session.query( + Hugothemes_from_gitlab).filter_by(name=theme).one() if gitlab_theme.gitlab_id is None: response = get(f"https://{gitlab_theme.url}") if response.status_code == 200: gitlab_theme.gitlab_id = match.search(response.text).group(2) session.commit() if response.status_code == 404: - print(response.status_code, get_gitlab_project_ids.__name__, theme) + print( + response.status_code, + get_gitlab_project_ids.__name__, theme) print(response.status_code, get_gitlab_project_ids.__name__) @@ -194,7 +218,8 @@ def get_commit_info_for_hugo_themes(): theme_names_from_github = get_github_themes_name_list() for hugo_theme in theme_names_from_github: theme = session.query(Hugothemes).filter_by(name=hugo_theme).one() - api_call_url = f'https://api.github.com/repos/{theme.name}/commits/{theme.default_branch}' + api_call_url = 'https://api.github.com/repos/' + api_call_url += f'{theme.name}/commits/{theme.default_branch}' if theme.ETag is not None: headers['If-None-Match'] = theme.ETag @@ -213,14 +238,18 @@ def get_commit_info_for_hugo_themes(): theme.ETag = response.headers['ETag'].lstrip('W/') result = response.json() theme.commit_date = result['commit']['author']['date'] - theme.commit_date_in_seconds = timegm(strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) + theme.commit_date_in_seconds = timegm( + strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) theme.commit_sha = result['commit']['tree']['sha'] session.commit() elif response.status_code == 403: - print(response.status_code, get_commit_info_for_hugo_themes.__name__) + print( + response.status_code, get_commit_info_for_hugo_themes.__name__) quit() elif response.status_code == 404: - print(response.status_code, get_commit_info_for_hugo_themes.__name__, hugo_theme) + print( + response.status_code, + get_commit_info_for_hugo_themes.__name__, hugo_theme) print(response.status_code, get_commit_info_for_hugo_themes.__name__) @@ -229,23 +258,36 @@ def get_commit_info_for_hugo_themes_from_gitlab(): theme_names_from_gitlab = get_gitlab_themes_name_list() # match = re.compile(r'(\.\d{3})(Z$)') for hugo_theme in theme_names_from_gitlab: - theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() - api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}/repository/commits/{theme.default_branch}' + theme = session.query( + Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() + api_call_url = 'https://gitlab.com/api/v4/projects/' + api_call_url += f'{theme.gitlab_id}/repository/' + api_call_url += f'commits/{theme.default_branch}' response = get(api_call_url) if response.status_code == 200: result = response.json() - # theme.commit_date = (match.sub(r'\2', result['created_at']))[0:19] + 'Z' + # theme.commit_date = (match.sub(r'\2', result[ + # 'created_at']))[0:19] + 'Z' theme.commit_date = result['created_at'][0:19] + 'Z' - theme.commit_date_in_seconds = timegm(strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) + theme.commit_date_in_seconds = timegm( + strptime(theme.commit_date, '%Y-%m-%dT%H:%M:%SZ')) theme.commit_sha = result['id'] session.commit() elif response.status_code == 403: - print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__) + print( + response.status_code, + get_commit_info_for_hugo_themes_from_gitlab.__name__) quit() elif response.status_code == 404: - print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__, hugo_theme) - print(response.status_code, get_commit_info_for_hugo_themes_from_gitlab.__name__) + print( + response.status_code, + get_commit_info_for_hugo_themes_from_gitlab.__name__, + hugo_theme + ) + print( + response.status_code, + get_commit_info_for_hugo_themes_from_gitlab.__name__) def get_repo_info_for_hugo_themes(): @@ -275,7 +317,9 @@ def get_repo_info_for_hugo_themes(): print(response.status_code, get_repo_info_for_hugo_themes.__name__) quit() elif response.status_code == 404: - print(response.status_code, get_repo_info_for_hugo_themes.__name__, hugo_theme) + print( + response.status_code, + get_repo_info_for_hugo_themes.__name__, hugo_theme) print(response.status_code, get_repo_info_for_hugo_themes.__name__) @@ -283,7 +327,8 @@ def get_repo_info_for_hugo_themes_from_gitlab(): session = sessionmaker(bind=engine)() theme_names_from_gitlab = get_gitlab_themes_name_list() for hugo_theme in theme_names_from_gitlab: - theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() + theme = session.query(Hugothemes_from_gitlab).filter_by( + name=hugo_theme).one() api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}' response = get(api_call_url) if response.status_code == 200: @@ -294,11 +339,17 @@ def get_repo_info_for_hugo_themes_from_gitlab(): theme.default_branch = result['default_branch'] session.commit() elif response.status_code == 403: - print(response.status_code, get_repo_info_for_hugo_themes_from_gitlab.__name__) + print( + response.status_code, + get_repo_info_for_hugo_themes_from_gitlab.__name__) quit() elif response.status_code == 404: - print(response.status_code, get_repo_info_for_hugo_themes_from_gitlab.__name__, hugo_theme) - print(response.status_code, get_repo_info_for_hugo_themes_from_gitlab.__name__) + print( + response.status_code, + get_repo_info_for_hugo_themes_from_gitlab.__name__, hugo_theme) + print( + response.status_code, + get_repo_info_for_hugo_themes_from_gitlab.__name__) def get_theme_dot_toml_for_each_hugo_themes(): @@ -310,7 +361,8 @@ def get_theme_dot_toml_for_each_hugo_themes(): theme_toml = 'wowchemy/theme.toml' else: theme_toml = 'theme.toml' - api_call_url = f"https://api.github.com/repos/{theme.name}/contents/{theme_toml}" + api_call_url = "https://api.github.com/repos/" + api_call_url += f"{theme.name}/contents/{theme_toml}" if theme.themes_toml_ETag is not None: headers['If-None-Match'] = theme.themes_toml_ETag @@ -328,51 +380,75 @@ def get_theme_dot_toml_for_each_hugo_themes(): theme.themes_toml_content = result['content'] session.commit() elif response.status_code == 403: - print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__) + print( + response.status_code, + get_theme_dot_toml_for_each_hugo_themes.__name__) quit() elif response.status_code == 404: - print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__, hugo_theme) - print(response.status_code, get_theme_dot_toml_for_each_hugo_themes.__name__) + print( + response.status_code, + get_theme_dot_toml_for_each_hugo_themes.__name__, hugo_theme) + print( + response.status_code, + get_theme_dot_toml_for_each_hugo_themes.__name__) def get_theme_dot_toml_for_each_hugo_themes_from_gitlab(): session = sessionmaker(bind=engine)() theme_names_from_gitlab = get_gitlab_themes_name_list() for hugo_theme in theme_names_from_gitlab: - theme = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() - api_call_url = f'https://gitlab.com/api/v4/projects/{theme.gitlab_id}/repository/files/theme.toml?ref={theme.default_branch}' + theme = session.query( + Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() + api_call_url = 'https://gitlab.com/api/v4/projects/' + api_call_url += f'{theme.gitlab_id}/repository/files/' + api_call_url += f'theme.toml?ref={theme.default_branch}' response = get(api_call_url) if response.status_code == 200: result = response.json() theme.themes_toml_content = result['content'] session.commit() elif response.status_code == 403: - print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) + print( + response.status_code, + get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) quit() elif response.status_code == 404: - print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__, hugo_theme) - print(response.status_code, get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) + print( + response.status_code, + get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__, + hugo_theme) + print( + response.status_code, + get_theme_dot_toml_for_each_hugo_themes_from_gitlab.__name__) def coalesce_themes(): session = sessionmaker(bind=engine)() theme_names_from_gitlab = get_gitlab_themes_name_list() for hugo_theme in theme_names_from_gitlab: - htfgitlab = session.query(Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() + htfgitlab = session.query( + Hugothemes_from_gitlab).filter_by(name=hugo_theme).one() theme = session.query(Hugothemes).filter_by(name=hugo_theme).first() if theme is None: session.add(Hugothemes( - name=hugo_theme, url=htfgitlab.url, commit_sha=htfgitlab.commit_sha, - commit_date=htfgitlab.commit_date, commit_date_in_seconds=htfgitlab.commit_date_in_seconds, - stargazers_count=htfgitlab.star_count, themes_toml_content=htfgitlab.themes_toml_content + name=hugo_theme, url=htfgitlab.url, + commit_sha=htfgitlab.commit_sha, + commit_date=htfgitlab.commit_date, + commit_date_in_seconds=htfgitlab.commit_date_in_seconds, + stargazers_count=htfgitlab.star_count, + themes_toml_content=htfgitlab.themes_toml_content, )) else: if theme.url != htfgitlab.url: theme.url = htfgitlab.url - if theme.commit_sha != htfgitlab.commit_sha: theme.commit_sha = htfgitlab.commit_sha - if theme.commit_date != htfgitlab.commit_date: theme.commit_date = htfgitlab.commit_date - if theme.commit_date_in_seconds != htfgitlab.commit_date_in_seconds: - theme.commit_date_in_seconds = htfgitlab.commit_date_in_seconds - if theme.stargazers_count != htfgitlab.star_count: theme.stargazers_count = htfgitlab.star_count + if theme.commit_sha != htfgitlab.commit_sha: + theme.commit_sha = htfgitlab.commit_sha + if theme.commit_date != htfgitlab.commit_date: + theme.commit_date = htfgitlab.commit_date + htfgitlab_cdis = htfgitlab.commit_date_in_seconds + if theme.commit_date_in_seconds != htfgitlab_cdis: + theme.commit_date_in_seconds = htfgitlab_cdis + if theme.stargazers_count != htfgitlab.star_count: + theme.stargazers_count = htfgitlab.star_count theme.themes_toml_content = htfgitlab.themes_toml_content session.commit() @@ -405,10 +481,13 @@ def parse_themes_toml_for_each_hugo_themes(): if 'tags' in theme_toml: if len(theme_toml['tags']) > 0: corrected_tags = get_corrected_tags(theme_toml['tags']) - theme_tags = [tag.lower() for tag in corrected_tags if len(tag) > 1] - if theme.num_tags != len(theme_tags): theme.num_tags = len(theme_tags) + theme_tags = [ + tag.lower() for tag in corrected_tags if len(tag) > 1] + if theme.num_tags != len(theme_tags): + theme.num_tags = len(theme_tags) if theme.num_tags > 0: - if theme.tags_list != str(theme_tags): theme.tags_list = str(theme_tags) + if theme.tags_list != str(theme_tags): + theme.tags_list = str(theme_tags) else: if theme.tags_list is not None: theme.tags_list = None else: @@ -419,14 +498,20 @@ def parse_themes_toml_for_each_hugo_themes(): if theme.num_tags != 0: theme.num_tags = 0 if 'features' in theme_toml: if len(theme_toml['features']) > 0: - theme_features = [feature.lower() for feature in theme_toml['features'] if len(feature) > 1] - if theme.num_features != len(theme_features): theme.num_features = len(theme_features) + theme_features = [ + feature.lower() for feature in theme_toml[ + 'features'] if len(feature) > 1] + if theme.num_features != len(theme_features): + theme.num_features = len(theme_features) if theme.num_features > 0: - if theme.features_list != str(theme_features): theme.features_list = str(theme_features) + if theme.features_list != str(theme_features): + theme.features_list = str(theme_features) else: - if theme.features_list is not None: theme.features_list = None + if theme.features_list is not None: + theme.features_list = None else: - if theme.features_list is not None: theme.features_list = None + if theme.features_list is not None: + theme.features_list = None if theme.num_features != 0: theme.num_features = 0 else: if theme.features_list is not None: theme.features_list = None @@ -474,12 +559,17 @@ def generate_report(): 'url': f'https://{theme.url}', 'short_name': theme.name.split('/')[1], 'num_stars': theme.stargazers_count, - 'tags': literal_eval(theme.tags_list) if theme.tags_list is not None else [], - 'features': literal_eval(theme.features_list) if theme.features_list is not None else [], - 'license': theme.theme_license if theme.theme_license is not None else '', + 'tags': literal_eval( + theme.tags_list) if theme.tags_list is not None else [], + 'features': literal_eval( + theme.features_list + ) if theme.features_list is not None else [], + 'license': theme.theme_license if + theme.theme_license is not None else '', 'min_ver': theme.min_ver if theme.min_ver is not None else '', 'desc': theme.desc if theme.desc is not None else '', - 'cname': theme.cname if theme.cname is not None else theme.name.split('/')[1], + 'cname': theme.cname if + theme.cname is not None else theme.name.split('/')[1], } for theme in session.query(Hugothemes).all() ] output = template.render(themes=hugo_themes) @@ -492,7 +582,9 @@ if __name__ == "__main__": ''' or test with ` - python3 -c'import rank_hugo_themes; rank_hugo_themes.parse_themes_toml_for_each_hugo_themes() ; rank_hugo_themes.generate_report()' + python3 -c'import rank_hugo_themes; rank_hugo_themes + .parse_themes_toml_for_each_hugo_themes() + ; rank_hugo_themes.generate_report()' ` ''' dedup_database()