Add ability to update supply data from web automatically. Will pick up price changes and allow for fitting to best size order from all sources.

refresh.py

@@ -0,0 +1,121 @@
+from argparse import ArgumentParser,ArgumentTypeError
+import csv
+import asyncio
+from pyppeteer import launch
+from pyquery import PyQuery as pq
+import json
+import math
+from os import linesep as EOL
+
+ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0'
+sources = ['artcraft', 'photographers_formulary', 'bandh']
+csvFile = './sources.csv'
+csvOutput = './supply.csv'
+
+def is_source (source) :
+    if source in sources :
+        return source
+    raise ArgumentTypeError('Not in list')
+
+async def artcraft (url, chemical) :
+    browser = await launch()
+    page = await browser.newPage()
+    await page.setUserAgent(ua)
+    print(url)
+    await page.goto(url)
+    await asyncio.sleep(1)
+    html = await page.content()
+#    with open('./test.html', 'w') as file :
+#       file.write(html)
+    data = parse_artcraft_product(html)
+    await asyncio.sleep(5)
+    for variant in data : 
+        #print(variant)
+        variant_url = f'{url}?variant={variant["id"]}'
+        print(variant_url)
+        await page.goto(variant_url)
+        html = await page.content()
+        variant['url'] = variant_url
+        variant['price'] = parse_artcraft_price(html)
+    await browser.close()
+    #print(html)
+    for v in data :
+        artcraft_line(chemical, v)
+
+def artcraft_line (chemical, v) :
+    #chemical,url,grams,milliliters,price
+    line = f'{chemical},{v["url"]},{v["weight"]},,{v["price"]}{EOL}'
+    print(line)
+    with open(csvOutput, 'a') as file :
+        file.write(line)
+
+async def artcraft_test (url, chemical) :
+    print(url)
+    with open('./test.html', 'r') as file :
+        html = file.read()
+    data = parse_artcraft_product(html)
+    print(data)
+
+def parse_artcraft_id (text) :
+    return text.split('-')[2].split('_')[0]
+
+def parse_artcraft_weight (text) :
+    text = text.lower()
+    text = text.replace('-1/2', '.5')
+    if 'pound' in text :
+        val = float(text.split(' ')[0]) * 453.592
+    elif 'gram' in text :
+        val = float(text.split(' ')[0])
+    else :
+        val = None
+    return val
+
+def parse_artcraft_price (html) :
+#    with open('./test.html', 'w') as file :
+#        file.write(html)
+    d = pq(html)
+    price = None
+    for p in d('span.price-item').items() :
+        price = p.text()
+        break
+    #print(price)
+    if price is None :
+        return None
+    dollars = price.replace('$', '').replace(' ', '').replace('USD', '').strip()
+    #print(dollars)
+    cents = math.ceil(float(dollars) * 100.0)
+    return cents
+
+def parse_artcraft_product (html) :
+    d = pq(html)
+    data = []
+    scr = d('variant-radios script').text()
+    objs = json.loads(scr)
+    for o in objs:
+        data.append({
+            'id' : o['id'],
+            'weight' : parse_artcraft_weight(o['title'])
+        })
+    return data
+
+async def main () :
+    parser = ArgumentParser(description='Refresh prices from sources')
+    parser.add_argument('-s', '--source', type=is_source, required=False, default=None, help='Only run on single source')
+    args = parser.parse_args()
+
+    with open(csvOutput, 'w') as file :
+        file.write('chemical,url,grams,milliliters,price' + EOL)
+
+    with open(csvFile, newline='') as csvfile:
+        reader = csv.reader(csvfile, delimiter=',', quotechar='|')
+        for row in reader:
+            source = row[0]
+            chemical = row[1]
+            url = row[2]
+            if source == 'source' or (args.source is not None and source != args.source) :
+                continue
+            if source == 'artcraft' :
+                await artcraft(url, chemical)
+
+if __name__ == '__main__' : 
+    asyncio.get_event_loop().run_until_complete(main())

requirements.txt

@@ -0,0 +1,2 @@
+pyppeteer
+pyquery

sources.csv

@@ -0,0 +1,9 @@
+source,chemical,url
+artcraft,Sodium Sulfite (anhydrous),https://artcraftchemicals.com/products/sodium-sulfite-anhy-part-1416
+artcraft,Sodium Carbonate (monohydrate),https://artcraftchemicals.com/products/sodium-carbonate-mono-part-1340
+artcraft,Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234
+artcraft,Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200
+artcraft,Metol,https://artcraftchemicals.com/products/metol-part-1185
+artcraft,Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180
+artcraft,Sodium Metabisulfite,https://artcraftchemicals.com/products/sodium-metabisulfite-part-1375
+artcraft,Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435

supply.csv

@@ -1,15 +1,24 @@
 chemical,url,grams,milliliters,price
-Sodium Sulfite (anhydrous),https://www.bhphotovideo.com/c/product/124092-REG/Photographers_Formulary_10_1340_1LB_Sodium_Sulfite_1.html,453.592,,695
+Sodium Sulfite (anhydrous),https://artcraftchemicals.com/products/sodium-sulfite-anhy-part-1416?variant=42892710904055,2267.96,,1811
-Sodium Carbonate (monohydrate),https://www.bhphotovideo.com/c/product/124044-REG/Photographers_Formulary_10_1190_1LB_Sodium_Carbonate_Monohydrate.html,453.592,,995
+Sodium Sulfite (anhydrous),https://artcraftchemicals.com/products/sodium-sulfite-anhy-part-1416?variant=42892710969591,9071.84,,5457
-Potassium Bromide,https://www.bhphotovideo.com/c/product/123938-REG/Photographers_Formulary_10_0930_1LB_Potassium_Bromide_1.html,453.592,,2195
+Sodium Sulfite (anhydrous),https://artcraftchemicals.com/products/sodium-sulfite-anhy-part-1416?variant=42892711035127,453.592,,663
-Phenidone,https://www.bhphotovideo.com/c/product/123915-REG/Photographers_Formulary_10_0870_100G_Phenidone_100_Grams.html,100,,2695
+Sodium Carbonate (monohydrate),https://artcraftchemicals.com/products/sodium-carbonate-mono-part-1340?variant=42892721979639,453.592,,785
-Metol,https://www.bhphotovideo.com/c/product/123903-REG/Photographers_Formulary_10_0770_100G_Metol_Elon_100.html,100,,1695
+Sodium Carbonate (monohydrate),https://artcraftchemicals.com/products/sodium-carbonate-mono-part-1340?variant=42892722012407,2041.164,,2922
-Hydroquinone,https://www.bhphotovideo.com/c/product/123903-REG/Photographers_Formulary_10_0770_100G_Metol_Elon_100.html,100,,1695
+Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234?variant=42892792266999,100.0,,791
-Glycin,https://www.bhphotovideo.com/c/product/123868-REG/Photographers_Formulary_10_0610_100G_Glycin_N_Parahydroxyphenyl.html,100,,2395
+Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234?variant=42892792299767,250.0,,1265
-Borax,https://www.bhphotovideo.com/c/product/123755-REG/Photographers_Formulary_10_0260_1LB_Borax_1_lb.html,453.592,,695
+Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234?variant=42892792332535,453.592,,2036
-Hydroquinone,https://www.bhphotovideo.com/c/product/123887-REG/Photographers_Formulary_10_0670_100G_Hydroquinone_100_Grams.html,100,,895
+Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234?variant=43227267006711,2267.96,,8058
-Sodium Metabisulfite,https://www.bhphotovideo.com/c/product/124063-REG/Photographers_Formulary_10_1280_100G_Sodium_Metabisulfite_100g.html,100,,595
+Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200?variant=42892799639799,50.0,,1474
-Sodium Metaborate,https://www.bhphotovideo.com/c/product/124068-REG/Photographers_Formulary_10_1285_1LB_Sodium_Metaborate_Balanced_Alkali.html,453.592,,1195
+Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200?variant=42892799672567,100.0,,2428
-Potassium Metabisulfite,https://morewinemaking.com/products/potassium-metabisulfite.html,453.592,,1199
+Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200?variant=42892799705335,250.0,,5319
-Sodium Hydroxide,https://www.amazon.com/Sodium-Hydroxide-Grade-Caustic-Pound/dp/B07KNR9SVF,907.184,,1799
+Metol,https://artcraftchemicals.com/products/metol-part-1185?variant=42892802851063,100.0,,1831
-p-Aminophenol Hydrochloride,https://www.sigmaaldrich.com/US/en/product/sial/09170,100,,10300
+Metol,https://artcraftchemicals.com/products/metol-part-1185?variant=42892802883831,250.0,,3973
+Metol,https://artcraftchemicals.com/products/metol-part-1185?variant=42892802916599,453.592,,6217
+Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180?variant=42892822675703,453.592,,2346
+Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180?variant=42892822708471,250.0,,1479
+Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180?variant=43085712654583,2267.96,,9410
+Sodium Metabisulfite,https://artcraftchemicals.com/products/sodium-metabisulfite-part-1375?variant=42892717326583,453.592,,729
+Sodium Metabisulfite,https://artcraftchemicals.com/products/sodium-metabisulfite-part-1375?variant=42892717359351,2267.96,,2474
+Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435?variant=42892693602551,453.592,,709
+Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435?variant=42892693635319,2267.96,,2036
+Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435?variant=42892693668087,8618.248,,5809

supply_old.csv

@@ -0,0 +1,18 @@
+chemical,url,grams,milliliters,price
+Sodium Sulfite (anhydrous),https://www.bhphotovideo.com/c/product/124092-REG/Photographers_Formulary_10_1340_1LB_Sodium_Sulfite_1.html,453.592,,695
+Sodium Carbonate (monohydrate),https://www.bhphotovideo.com/c/product/124044-REG/Photographers_Formulary_10_1190_1LB_Sodium_Carbonate_Monohydrate.html,453.592,,995
+Potassium Bromide,https://www.bhphotovideo.com/c/product/123938-REG/Photographers_Formulary_10_0930_1LB_Potassium_Bromide_1.html,453.592,,2195
+Phenidone,https://www.bhphotovideo.com/c/product/123915-REG/Photographers_Formulary_10_0870_100G_Phenidone_100_Grams.html,100,,2695
+Metol,https://www.bhphotovideo.com/c/product/123903-REG/Photographers_Formulary_10_0770_100G_Metol_Elon_100.html,100,,1695
+Hydroquinone,https://www.bhphotovideo.com/c/product/123903-REG/Photographers_Formulary_10_0770_100G_Metol_Elon_100.html,100,,1695
+Glycin,https://www.bhphotovideo.com/c/product/123868-REG/Photographers_Formulary_10_0610_100G_Glycin_N_Parahydroxyphenyl.html,100,,2395
+Borax,https://www.bhphotovideo.com/c/product/123755-REG/Photographers_Formulary_10_0260_1LB_Borax_1_lb.html,453.592,,695
+Borax,https://artcraftchemicals.com/products/borax-part-1070,453.592,,663
+Hydroquinone,https://www.bhphotovideo.com/c/product/123887-REG/Photographers_Formulary_10_0670_100G_Hydroquinone_100_Grams.html,100,,895
+Sodium Metabisulfite,https://www.bhphotovideo.com/c/product/124063-REG/Photographers_Formulary_10_1280_100G_Sodium_Metabisulfite_100g.html,100,,595
+Sodium Metaborate,https://www.bhphotovideo.com/c/product/124068-REG/Photographers_Formulary_10_1285_1LB_Sodium_Metaborate_Balanced_Alkali.html,453.592,,1195
+Sodium Metaborate,https://artcraftchemicals.com/products/sodium-metaborate-bal-alkali-part-1385,453.592,,1015
+Potassium Metabisulfite,https://morewinemaking.com/products/potassium-metabisulfite.html,453.592,,1199
+Sodium Hydroxide,https://www.amazon.com/Sodium-Hydroxide-Grade-Caustic-Pound/dp/B07KNR9SVF,907.184,,1799
+Sodium Hydroxide,https://artcraftchemicals.com/products/sodium-hydroxide-part-1759,453.592,,1015
+p-Aminophenol Hydrochloride,https://www.sigmaaldrich.com/US/en/product/sial/09170,100,,10300