Browse Source

Fix: Minor changes to admin front+backend to change IP Location API and make it less likely to get blacklisted

Sebastian Kreisel 7 months ago
parent
commit
1d21626e92

+ 22 - 14
TODO.txt

@@ -1,20 +1,27 @@
 -- Urgent
 ------------------------------------------------------------------------------
-* Backend: Polish list view in rubble (add selection etc)
-* Backend: Make consistent and decent rubble empty view
-* Frontend: Restrict ip loc requests to 300 per day via localstoreage
-* CSS+Parsing: Improve HTML5 video appearance
 
-
+-- Christmas Update
 ------------------------------------------------------------------------------
-* Backend: Instead of removing on "delete" change post type
-* CSS: Improve on h2 fontsize and styling
-* Files: Permissions on uploaded files
-* Edit: Fix Parsing Bugs
-* Edit: Make parsing response work again
-* Docker: Find a way to output stdcout from docker
-* Edit: On acc/ptype-change re-fetch
+* Big-Feature: Backend: Content type and View for email transcripts
+* Big-Feature: Imageviewer: Rework + add paging to avoid huge amount requests
+
+* Small-Feature: Backend: Polish list view in rubble (add selection etc)
+* Small-Feature: Backend: Make consistent and decent rubble empty view
+* Small-Feature: Backend: Instead of removing on "delete" change post type
+* Small-Feature: Parsing: Add video to js-parser
+* Small-Feature: Backend+Frontend: Rework IP location
+* Small-Feature: Frontend: Edit: On acc/ptype-change re-fetch
+
+* Improvement: CSS+Parsing: Improve HTML5 video appearance
+* Improvement: Frontend: Edit: Make parsing response work again
+* Improvement: CSS: Improve on h2 fontsize and styling
+
+* Fix: Fontend: Inverstigate why sha1 stuff still shows up (?)
+* Fix: Parsing: Spacing in \center\minipage etc
+* Fix: Backend: Fix list-view math title height
+* Fix: Frontend: Edit: Fix Parsing Bugs
+* Fix: Frontend: Edit: Investigate lag in certain long posts / keystrokes
 
 * Style Refact: XHR Prodcedure
 * Style Refact: Rework Promise flow
@@ -22,8 +29,8 @@
 * Style Refact: Catch all possible IO Exceptions in Haskell
 * Style Refact: filedirectory CSS+HTML mess
 
-* Feature: Backend: Content type and View for email transcripts
-
+* Misc: Files: Permissions on uploaded files
+* Misc: Docker: Find a way to output stdcout from docker
 * Misc: Make consistent color scheme for all of admin
 * Misc: Backend: Decide if admin access is 5 or 10 and make consistent
 * Misc: Backend: In serveAdminSite use real access mod not magic 10

+ 3 - 1
elfcom-backend/src/Endpoint/AccessLog.hs

@@ -54,8 +54,10 @@ accessLogLocHandler (AccessLogLocReq iptext) = do
 
 fetchIpLocation :: T.Text -> IO (Maybe IpLocation)
 fetchIpLocation iptext = do
-  let urll = "https://ipapi.co/" <> iptext <> "/json/"
+  let urll = "http://ip-api.com/json/" <> iptext <>
+             "?fields=query,city,country,countryCode"
   result <- httpLBS (parseRequest_ $ T.unpack urll)
+  --putStrLn $ show (getResponseBody result)
   return (decode $ getResponseBody result)
 
 data AccessLogFetchReq = AccessLogFetchReq {

+ 2 - 2
elfcom-backend/src/Model/AccessLog.hs

@@ -51,10 +51,10 @@ data IpLocation = IpLocation {
                 deriving (Show, Eq, Generic, ToJSON)
 
 instance FromJSON IpLocation where
-  parseJSON (Object v) = IpLocation <$> v .: "ip"
+  parseJSON (Object v) = IpLocation <$> v .: "query"
                          <*> v .: "city"
-                         <*> v .: "country_name"
                          <*> v .: "country"
+                         <*> v .: "countryCode"
 
 -- -------------------------------------------------------------------------
 

+ 42 - 24
elfcom-frontend/admin/log.js

@@ -7,7 +7,10 @@ import SelTree from "../seleck.js";
 var Log = {
   init: init,
   doFetch: doFetch,
+  checkIpLocationQueue: checkIpLocationQueue,
   selTree: null,
+  locQueue: [],
+  rateLimit: 150,
 };
 export default Log;
 
@@ -25,7 +28,8 @@ function init(sobjs) {
   if(locValid === null) {
     window.localStorage.setItem("locValid", ("" + (new Date()).getTime()));
   }
-  window.setInterval(resetLocCounter, 65 * 1000);
+  // reset the loc counter (limiting requests) every minute
+  window.setInterval(resetLocCounter, 60 * 1000);
   var start = new Date();
   start.setHours(0, 0, 0, 0);
   var end = new Date();
@@ -186,31 +190,45 @@ function checkLocationStorage(ip) {
 }
 
 function enqueueLocation(ip, locCont) {
+  Log.locQueue.push([ip, locCont]);
+}
+
+function checkIpLocationQueue() {
   var locCounter = parseInt(window.localStorage.getItem("locCounter"));
-  if(locCounter < 300) {
-    window.localStorage.setItem("locCounter", ("" + (locCounter + 1)));
-    var jsonObj = {
-      "locIp": ip,
-    };
-    Common.sendJson("/api/admin/accesslog/loc", jsonObj)
-      .then(res => res.json(),
-            () => { Common.fetchError("Fetch"); reject("ajax failed"); })
-      .then(function(json) {
-        if(json.tag === "AccessLogLocOkay") {
-          var loc = {
-            cc: json.locLoc.locCountryCode,
-            city: json.locLoc.locCity
-          };
-          console.log(loc);
-          var locDict = JSON.parse(window.localStorage.getItem("locDict"));
-          locDict[ip] = loc;
-          window.localStorage.setItem("locDict", JSON.stringify(locDict));
-          setLocationContainer(loc, locCont);
-        } else {
-          Common.serverError("Fetch"); reject("server failed");
-        }
-      }, () => { Common.jsonError("Fetch"); reject("json failed"); });
+  if(Log.locQueue.length == 0) {
+    return;
+  }
+  if(locCounter >= Log.rateLimit) {
+    Common.generalError("Rate Limit reached");
   }
+  // Get first time off queue and pack it to the end. If this function
+  // succeedes it will be removed off the end
+  var entry = Log.locQueue.shift();
+  Log.locQueue.push(entry);
+  var ip = entry[0];
+  var locCont = entry[1];
+  window.localStorage.setItem("locCounter", ("" + (locCounter + 1)));
+  var jsonObj = {
+    "locIp": ip,
+  };
+  Common.sendJson("/api/admin/accesslog/loc", jsonObj)
+    .then(res => res.json(),
+          () => { Common.fetchError("Fetch"); reject("ajax failed"); })
+    .then(function(json) {
+      if(json.tag === "AccessLogLocOkay") {
+        var loc = {
+          cc: json.locLoc.locCountryCode,
+          city: json.locLoc.locCity
+        };
+        var locDict = JSON.parse(window.localStorage.getItem("locDict"));
+        locDict[ip] = loc;
+        window.localStorage.setItem("locDict", JSON.stringify(locDict));
+        setLocationContainer(loc, locCont);
+        Log.locQueue.pop(); // remove entry from the back of the queue
+      } else {
+        Common.serverError("Fetch"); reject("server failed");
+      }
+    }, () => { Common.jsonError("Fetch"); reject("json failed"); });
 }
 
 function setLocationContainer(loc, locCont) {

+ 3 - 2
elfcom-frontend/admin/site_admin.js

@@ -5,7 +5,6 @@ import Log from "./log.js";
 
 
 document.addEventListener("DOMContentLoaded", function() {
-
   var sobjs = {
     "logEl": document.getElementById("admin-log-content"),
     "seaEl": document.getElementById("admin-log-search"),
@@ -15,7 +14,6 @@ document.addEventListener("DOMContentLoaded", function() {
   Common.init(sobjs);
   registerEventHandlers(sobjs);
   Log.init(sobjs);
-
 });
 
 function registerEventHandlers(sobjs) {
@@ -24,4 +22,7 @@ function registerEventHandlers(sobjs) {
       Log.doFetch(sobjs);
     }
   };
+  window.setInterval(function() {
+    Log.checkIpLocationQueue();
+  }, 1000);
 }