Merge branch 'async-space-load' into 'master'

load space apis asynchronously

See merge request mamu/led_karte!1

draw_map.py

@@ -1,5 +1,10 @@
 #!/usr/bin/python
-import requests, json, pickle, os.path, svgwrite, math
+import requests
+import json
+import pickle
+import os.path
+import svgwrite
+import math
 
 URL = "https://directory.spaceapi.io/"
 NORTHERNMOST = 55.05
@@ -15,11 +20,13 @@ blacklist = ["Chaostreff Salzburg", "DevLoL", "CCC Basel", "Chaostreff Zürich",
              "ChaosStuff", "Level2", "Bastli", "Maakplek", "TkkrLab", "Hack42",
              "Hackerspace Nijmegen", "TDvenlo", "ACKspace"]
 
+
 def dist(n1, n2):
     y = n1[0] - n2[0]
     x = n1[1] - n2[1]
     return math.sqrt(math.pow(x, 2) + math.pow(y, 2))
 
+
 def conflict(targetlist, node):
     returner = None
     for element in targetlist:
@@ -28,6 +35,7 @@ def conflict(targetlist, node):
             continue
     return returner
 
+
 def merge(n1, n2):
     lat = (n1[0] + n2[0]) / 2
     lon = (n1[1] + n2[1]) / 2
@@ -37,18 +45,19 @@ def merge(n1, n2):
     returner.append(n1[2] or n2[2])
     return returner
 
+
 if os.path.isfile('locations.bin'):
-    print ("using offline data...")
+    print("using offline data...")
     with open("locations.bin", "rb") as f:
         locations = pickle.load(f)
 else:
-    print ("offline data not available, downloading...,")
+    print("offline data not available, downloading...,")
-    r = requests.get(url = URL)
+    r = requests.get(url=URL)
     data = r.json()
     for space in data:
         spacerequest = None
         try:
-            spacerequest = requests.get(url = data[space], timeout=1)
+            spacerequest = requests.get(url=data[space], timeout=1)
         except requests.exceptions.RequestException as e:  # This is the correct syntax
             continue
         try:
@@ -61,13 +70,14 @@ else:
                 lon = spacedata["location"]["lon"]
                 if "state" in spacedata:
                     if "open" in spacedata["state"]:
-                        locations[space] = [float(lat), float(lon), spacedata["state"]["open"]]
+                        locations[space] = [float(lat), float(
+                            lon), spacedata["state"]["open"]]
 
     for space in locations:
-        print (space + " " +  str(locations[space]))
+        print(space + " " + str(locations[space]))
     with open("locations.bin", "wb") as f:
         pickle.dump(locations, f, pickle.HIGHEST_PROTOCOL)
-print ("Removing non-german points...")
+print("Removing non-german points...")
 german_locations = locations.copy()
 for space in locations:
     if locations[space][0] > NORTHERNMOST:
@@ -88,24 +98,25 @@ while german_locations:
     n1 = next(iter(german_locations))
     conflictnode = conflict(finallist, german_locations[n1])
     if conflictnode == None:
-        finallist.update({n1 : german_locations[n1]})
+        finallist.update({n1: german_locations[n1]})
         del german_locations[n1]
     else:
         mergenode = merge(german_locations[n1], finallist[conflictnode])
         del german_locations[n1]
         del finallist[conflictnode]
-        german_locations.update({"MERGED: " + n1 + " " + conflictnode : mergenode})
+        german_locations.update(
+            {"MERGED: " + n1 + " " + conflictnode: mergenode})
 
 for space in finallist:
     #print(space + " " + str(finallist[space][0]) + " " + str(finallist[space][1]))
     print(str(finallist[space][0]) + " " + str(finallist[space][1]) + " {" +
-            space + "}  Doorstatus: " + str(finallist[space][2]) )
+          space + "}  Doorstatus: " + str(finallist[space][2]))
 
 dwg = svgwrite.Drawing('svgwrite-example.svg', profile='tiny')
 dwg.add(svgwrite.image.Image(href="Karte_Deutschland.svg", size=(592, 801)))
-dwg.add(dwg.circle(center=(0,0), r=3, fill='black'))
+dwg.add(dwg.circle(center=(0, 0), r=3, fill='black'))
 for space in finallist:
-    ypoint = (801 - (((finallist[space][0] - SOUTHERNMOST) / YSPAN)* 801))
+    ypoint = (801 - (((finallist[space][0] - SOUTHERNMOST) / YSPAN) * 801))
     xpoint = ((finallist[space][1] - WESTERNMOST) / XSPAN) * 592
     if finallist[space][2]:
         dwg.add(dwg.circle(center=(xpoint, ypoint), r=5, fill='green'))

get_coordinates.py

@@ -1,5 +1,12 @@
 #!/usr/bin/python
-import requests, json, pickle, os.path, svgwrite, math
+import requests
+import json
+import pickle
+import os.path
+import svgwrite
+import math
+import sys
+import concurrent.futures
 
 URL = "https://directory.spaceapi.io/"
 NORTHERNMOST = 55.05
@@ -10,15 +17,17 @@ threshold = 0.10
 YSPAN = NORTHERNMOST - SOUTHERNMOST
 XSPAN = EASTERNMOST - WESTERNMOST
 locations = {}
-blacklist = ["Chaostreff Salzburg", "DevLoL", "CCC Basel", "Chaostreff Zürich",
+ignorelist = ["Chaostreff Salzburg", "DevLoL", "CCC Basel", "Chaostreff Zürich",
               "ChaosStuff", "Level2", "Bastli", "Maakplek", "TkkrLab", "Hack42",
               "Hackerspace Nijmegen", "TDvenlo", "ACKspace"]
 
+
 def dist(n1, n2):
     y = n1[0] - n2[0]
     x = n1[1] - n2[1]
     return math.sqrt(math.pow(x, 2) + math.pow(y, 2))
 
+
 def conflict(targetlist, node):
     returner = None
     for element in targetlist:
@@ -27,6 +36,7 @@ def conflict(targetlist, node):
             continue
     return returner
 
+
 def merge(n1, n2):
     lat = (n1[0] + n2[0]) / 2
     lon = (n1[1] + n2[1]) / 2
@@ -35,35 +45,46 @@ def merge(n1, n2):
     returner.append(lon)
     return returner
 
-if os.path.isfile('locations.bin'):
+
-    print ("using offline data...")
+def get_space_location(space):
-    with open("locations.bin", "rb") as f:
-        locations = pickle.load(f)
-else:
-    print ("offline data not available, downloading...,")
-    r = requests.get(url = URL)
-    data = r.json()
-    for space in data:
-        spacerequest = None
-        try:
-            spacerequest = requests.get(url = data[space], timeout=1)
-        except requests.exceptions.RequestException as e:  # This is the correct syntax
-            continue
     try:
+        spacerequest = requests.get(url=data[space], timeout=1)
         spacedata = spacerequest.json()
-        except json.JSONDecodeError as jde:
+    except requests.exceptions.RequestException as _:  # This is the correct syntax
-            continue
+        return
+    except json.JSONDecodeError as _:
+        return
     if "location" in spacedata:
         if "lat" in spacedata["location"]:
             lat = spacedata["location"]["lat"]
             lon = spacedata["location"]["lon"]
-                locations[space] = [float(lat), float(lon)]
+            return [float(lat), float(lon)]
+
+
+if os.path.isfile('locations.bin'):
+    print("using offline data...", file=sys.stderr)
+    with open("locations.bin", "rb") as f:
+        locations = pickle.load(f)
+else:
+    print("offline data not available, downloading...,", file=sys.stderr)
+    r = requests.get(url=URL)
+    data = r.json()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
+        futures = {}
+        for space in data:
+            futures[space] = executor.submit(get_space_location, space)
+        for space in data:
+            location = futures[space].result()
+            if location:
+                locations[space] = location
+    print("done loading space data", file=sys.stderr)
 
     for space in locations:
-        print (space + " " +  str(locations[space]))
+        print(space + " " + str(locations[space]))
     with open("locations.bin", "wb") as f:
         pickle.dump(locations, f, pickle.HIGHEST_PROTOCOL)
-print ("Removing non-german points...")
+
+print("Removing non-german points...", file=sys.stderr)
 german_locations = locations.copy()
 for space in locations:
     if locations[space][0] > NORTHERNMOST:
@@ -75,22 +96,22 @@ for space in locations:
     elif locations[space][1] > EASTERNMOST:
         del german_locations[space]
 
-for space in blacklist:
+for space in ignorelist:
     del german_locations[space]
 
-
 finallist = {}
 while german_locations:
     n1 = next(iter(german_locations))
     conflictnode = conflict(finallist, german_locations[n1])
     if conflictnode == None:
-        finallist.update({n1 : german_locations[n1]})
+        finallist.update({n1: german_locations[n1]})
         del german_locations[n1]
     else:
         mergenode = merge(german_locations[n1], finallist[conflictnode])
         del german_locations[n1]
         del finallist[conflictnode]
-        german_locations.update({"MERGED: " + n1 + " " + conflictnode : mergenode})
+        german_locations.update(
+            {"MERGED: " + n1 + " " + conflictnode: mergenode})
 
 for space in finallist:
     #print(space + " " + str(finallist[space][0]) + " " + str(finallist[space][1]))
@@ -99,9 +120,9 @@ for space in finallist:
 
 dwg = svgwrite.Drawing('svgwrite-example.svg', profile='tiny')
 dwg.add(svgwrite.image.Image(href="Karte_Deutschland.svg", size=(586, 793)))
-dwg.add(dwg.circle(center=(0,0), r=3, fill='black'))
 for space in finallist:
-    ypoint = (793 - (((finallist[space][0] - SOUTHERNMOST) / YSPAN)* 793))
+    ypoint = (793 - (((finallist[space][0] - SOUTHERNMOST) / YSPAN) * 793))
     xpoint = ((finallist[space][1] - WESTERNMOST) / XSPAN) * 586
     dwg.add(dwg.circle(center=(xpoint, ypoint), r=5, fill='green'))
+
 dwg.save()