checkpoint. beginning download process

portable_computing_toolkit_installer/tests/test_pages.py

@@ -0,0 +1,26 @@
+import json
+from urllib.request import urlopen
+import requests
+from pathlib import Path
+from parsel import Selector
+from w3lib.url import canonicalize_url
+
+import pytest
+
+json_url = "https://raw.githubusercontent.com/norweeg/portable-computing-toolkit-installer/initial_dev/portable_computing_toolkit_installer/resources/supported_tools.json"
+
+class TestPages(object):
+    def test_pages_accessible(self):
+        global json_url
+        with urlopen(json_url) as json_file:
+            json_data = json.load(json_file)
+        with requests.Session() as session:
+            for page in [canonicalize_url(item["homepage"]) for item in json_data]:
+                try:
+                    response = session.get(page, allow_redirects = True, verify = False)
+                except:
+                    response = session.get(page.replace("http", "https", 1), allow_redirects = True, verify = False)
+                assert response.status_code < 400
+                html = response.text
+                assert html
+                assert Selector(text = html, type="html").getall()