Add parsers for photographers formulary chemicals. Add sources to list. TODO: Add a one-off parser for artcraft

refresh.py

@@ -25,8 +25,8 @@ async def artcraft (url, chemical) :
     await page.goto(url)
     await asyncio.sleep(1)
     html = await page.content()
-#    with open('./test.html', 'w') as file :
+    #with open('./test.html', 'w') as file :
-#       file.write(html)
+    #   file.write(html)
     data = parse_artcraft_product(html)
     await asyncio.sleep(5)
     for variant in data : 
@@ -42,13 +42,6 @@ async def artcraft (url, chemical) :
     for v in data :
         artcraft_line(chemical, v)
 
-def artcraft_line (chemical, v) :
-    #chemical,url,grams,milliliters,price
-    line = f'{chemical},{v["url"]},{v["weight"]},,{v["price"]}{EOL}'
-    print(line)
-    with open(csvOutput, 'a') as file :
-        file.write(line)
-
 async def artcraft_test (url, chemical) :
     print(url)
     with open('./test.html', 'r') as file :
@@ -56,6 +49,13 @@ async def artcraft_test (url, chemical) :
     data = parse_artcraft_product(html)
     print(data)
 
+def artcraft_line (chemical, v) :
+    #chemical,url,grams,milliliters,price
+    line = f'{chemical},{v["url"]},{v["weight"]},,{v["price"]}{EOL}'
+    print(line)
+    with open(csvOutput, 'a') as file :
+        file.write(line)
+
 def parse_artcraft_id (text) :
     return text.split('-')[2].split('_')[0]
 
@@ -71,8 +71,8 @@ def parse_artcraft_weight (text) :
     return val
 
 def parse_artcraft_price (html) :
-#    with open('./test.html', 'w') as file :
+    #with open('./test.html', 'w') as file :
-#        file.write(html)
+    #   file.write(html)
     d = pq(html)
     price = None
     for p in d('span.price-item').items() :
@@ -98,13 +98,109 @@ def parse_artcraft_product (html) :
         })
     return data
 
+async def photographers_formulary (url, chemical) :
+    browser = await launch()
+    page = await browser.newPage()
+    await page.setUserAgent(ua)
+    print(url)
+    await page.goto(url)
+    await asyncio.sleep(1)
+    html = await page.content()
+    #with open('./test.html', 'w') as file :
+    #    file.write(html)
+    data = parse_photographers_formulary_product(html)
+    for b in data :
+        await asyncio.sleep(2)
+        print(b['weight'])
+        b['url'] = url
+        if b['type'] == 'li' :
+            index = b['index'] + 1
+            await page.click(f'.productOptionViewRadio ul li:nth-of-type({index}) label input')
+            await asyncio.sleep(2)
+            html = await page.content()
+            b['price'] = parse_photographers_formulary_price(html)
+            #print(b['price'])
+        elif b['type'] == 'select' :
+            await page.select('.productOptionViewSelect select', b['value'])
+            await asyncio.sleep(2)
+            html = await page.content()
+            b['price'] = parse_photographers_formulary_price(html)
+            #print(b['price'])
+    await browser.close()
+    for d in data :
+        photographers_formulary_line(chemical, d)
+
+async def photographers_formulary_test (url, chemical) :
+    print(url)
+    with open('./test.html', 'r') as file :
+        html = file.read()
+    data = parse_photographers_formulary_product(html)
+    print(data)
+
+def parse_photographers_formulary_product (html) :
+    d = pq(html)
+    data = []
+    i = 0
+    select = d('.productOptionViewSelect select')
+    if len(select) == 0 :
+        for b in d('.productOptionViewRadio ul li').items() :
+            data.append({
+                'index' : i,
+                'type' : 'li',
+                'weight' : parse_photographers_formulary_weight(b.find('span').text())
+            })
+            i+=1
+    elif len(select) == 1 :
+        for o in d('.productOptionViewSelect select option').items() :
+            if o.attr('value').strip() != '' :
+                data.append({
+                    'index' : i,
+                    'type' : 'select',
+                    'value' : o.attr('value'),
+                    'weight' : parse_photographers_formulary_weight(o.text())
+                })
+            i+=1
+    return data
+
+def parse_photographers_formulary_weight (text) :
+    parts = text.split(' ')
+    if parts[2] == 'g' :
+        return float(parts[1])
+    elif parts[2] == 'lb' :
+        return float(parts[1]) * 453.592
+    else :
+        return None
+
+def parse_photographers_formulary_price (html) :
+    #with open('./test.html', 'w') as file :
+    #   file.write(html)
+    d = pq(html)
+    price = None
+    for p in d('em.ProductPrice').items() :
+        price = p.text()
+        break
+    #print(price)
+    if price is None :
+        return None
+    dollars = price.replace('$', '').strip()
+    #print(dollars)
+    cents = math.ceil(float(dollars) * 100.0)
+    return cents
+
+def photographers_formulary_line (chemical, v) :
+    #chemical,url,grams,milliliters,price
+    line = f'{chemical},{v["url"]},{v["weight"]},,{v["price"]}{EOL}'
+    print(line)
+    with open(csvOutput, 'a') as file :
+        file.write(line)
+
 async def main () :
     parser = ArgumentParser(description='Refresh prices from sources')
     parser.add_argument('-s', '--source', type=is_source, required=False, default=None, help='Only run on single source')
     args = parser.parse_args()
 
-    with open(csvOutput, 'w') as file :
+    #with open(csvOutput, 'w') as file :
-        file.write('chemical,url,grams,milliliters,price' + EOL)
+    #    file.write('chemical,url,grams,milliliters,price' + EOL)
 
     with open(csvFile, newline='') as csvfile:
         reader = csv.reader(csvfile, delimiter=',', quotechar='|')
@@ -116,6 +212,8 @@ async def main () :
                 continue
             if source == 'artcraft' :
                 await artcraft(url, chemical)
+            elif source == 'photographers_formulary' :
+                await photographers_formulary(url, chemical)
 
 if __name__ == '__main__' : 
     asyncio.get_event_loop().run_until_complete(main())

sources.csv

@@ -6,4 +6,17 @@ artcraft,Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200
 artcraft,Metol,https://artcraftchemicals.com/products/metol-part-1185
 artcraft,Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180
 artcraft,Sodium Metabisulfite,https://artcraftchemicals.com/products/sodium-metabisulfite-part-1375
-artcraft,Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435
+artcraft,Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435
+photographers_formulary,Sodium Carbonate (monohydrate),https://stores.photoformulary.com/sodium-carbonate-monohydrate/
+photographers_formulary,Sodium Sulfite (anhydrous),https://stores.photoformulary.com/sodium-sulfite-anhydrous/
+photographers_formulary,Potassium Bromide,https://stores.photoformulary.com/potassium-bromide/
+photographers_formulary,Phenidone,https://stores.photoformulary.com/phenidone-class-6-1-ground-ups-only-choose-ups-ground-at-checkout/
+photographers_formulary,Metol,https://stores.photoformulary.com/metol-elon/
+photographers_formulary,Hydroquinone,https://stores.photoformulary.com/hydroquinone-class-6-1-ground-ups-only-choose-ups-ground-at-checkout/
+photographers_formulary,Glycin,https://stores.photoformulary.com/glycin/
+photographers_formulary,Borax,https://stores.photoformulary.com/borax/
+photographers_formulary,Sodium Metabisulfite,https://stores.photoformulary.com/sodium-metabisulfite/
+photographers_formulary,Sodium Metaborate,https://stores.photoformulary.com/sodium-metaborate/
+photographers_formulary,Potassium Metabisulfite,https://stores.photoformulary.com/potassium-metabisulfite/
+photographers_formulary,Sodium Hydroxide,https://stores.photoformulary.com/sodium-hydroxide-class-8-ground-ups-only-dea-form-required-choose-ups-ground-shipping-at-checkout/
+photographers_formulary,p-Aminophenol Hydrochloride,https://stores.photoformulary.com/p-aminophenol-hydrochloride/