webscraper is almost finished

webscraper/main.py

@@ -0,0 +1,60 @@
+from requests import get
+from requests.exceptions import RequestException
+from contextlib import closing
+from bs4 import BeautifulSoup
+import logging
+import json
+import time
+
+#universal vars
+
+# debug level
+frm = "%(asctime)-15s %(message)s"
+
+lvl = logging.WARNING
+
+
+
+
+#logging.basicConfig(level=logging.DEBUG)
+#logging.basicConfig(level=lvl,format=frm,file="log")
+
+
+def getReq(url):
+    try:
+        with closing(get(url, stream=True)) as resp:
+            if checkResponse(resp):
+                return resp.content
+            else:
+                return None
+
+    except RequestException as e:
+        logging.warning('error during requests to {} : {}'.format(url, str(e)))
+        return None
+
+
+def checkResponse(resp):
+    return (resp.status_code == 200
+            and resp.headers['Content-Type'].lower() is not None
+            and resp.headers['Content-Type'].lower().find('html') > -1)
+
+raw_html = getReq("https://answers.yahoo.com/dir/index")
+html = BeautifulSoup(raw_html, "html.parser")
+
+
+items = []
+
+
+for li in html.find_all('li', class_="ya-discover-tile ya-discover-tile-qn Bfc P-14 Bdbx-1g Bgc-w"):
+    arr = {}
+    arr['title'] = ' '.join(li.find('a', class_="title").text.split())
+    arr['category'] = ' '.join(li.find("div", class_="Clr-888 Fz-12 Lh-18").find('a', class_="Clr-b").text.split())
+    arr['description'] = ' '.join(li.find('div', class_="fullDesc Mah-130 Ovy-s Fz-13 Lh-18 Ol-n D-n").text.replace('\n', '').split())
+    if arr['description'].find('Best answer:') > -1:
+        arr['description'] = None
+
+    items.append(arr)
+
+
+with open('data/'+time.asctime().replace(":","-").replace(" ", "_"), "w") as f:
+    f.write(json.dumps(items))