Add ability to update supply data from web automatically. Will pick up price changes and allow for fitting to best size order from all sources.
This commit is contained in:
parent
ee4f7c6b8d
commit
bdea8474d3
|
|
@ -0,0 +1,121 @@
|
|||
from argparse import ArgumentParser,ArgumentTypeError
|
||||
import csv
|
||||
import asyncio
|
||||
from pyppeteer import launch
|
||||
from pyquery import PyQuery as pq
|
||||
import json
|
||||
import math
|
||||
from os import linesep as EOL
|
||||
|
||||
ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0'
|
||||
sources = ['artcraft', 'photographers_formulary', 'bandh']
|
||||
csvFile = './sources.csv'
|
||||
csvOutput = './supply.csv'
|
||||
|
||||
def is_source (source) :
|
||||
if source in sources :
|
||||
return source
|
||||
raise ArgumentTypeError('Not in list')
|
||||
|
||||
async def artcraft (url, chemical) :
|
||||
browser = await launch()
|
||||
page = await browser.newPage()
|
||||
await page.setUserAgent(ua)
|
||||
print(url)
|
||||
await page.goto(url)
|
||||
await asyncio.sleep(1)
|
||||
html = await page.content()
|
||||
# with open('./test.html', 'w') as file :
|
||||
# file.write(html)
|
||||
data = parse_artcraft_product(html)
|
||||
await asyncio.sleep(5)
|
||||
for variant in data :
|
||||
#print(variant)
|
||||
variant_url = f'{url}?variant={variant["id"]}'
|
||||
print(variant_url)
|
||||
await page.goto(variant_url)
|
||||
html = await page.content()
|
||||
variant['url'] = variant_url
|
||||
variant['price'] = parse_artcraft_price(html)
|
||||
await browser.close()
|
||||
#print(html)
|
||||
for v in data :
|
||||
artcraft_line(chemical, v)
|
||||
|
||||
def artcraft_line (chemical, v) :
|
||||
#chemical,url,grams,milliliters,price
|
||||
line = f'{chemical},{v["url"]},{v["weight"]},,{v["price"]}{EOL}'
|
||||
print(line)
|
||||
with open(csvOutput, 'a') as file :
|
||||
file.write(line)
|
||||
|
||||
async def artcraft_test (url, chemical) :
|
||||
print(url)
|
||||
with open('./test.html', 'r') as file :
|
||||
html = file.read()
|
||||
data = parse_artcraft_product(html)
|
||||
print(data)
|
||||
|
||||
def parse_artcraft_id (text) :
|
||||
return text.split('-')[2].split('_')[0]
|
||||
|
||||
def parse_artcraft_weight (text) :
|
||||
text = text.lower()
|
||||
text = text.replace('-1/2', '.5')
|
||||
if 'pound' in text :
|
||||
val = float(text.split(' ')[0]) * 453.592
|
||||
elif 'gram' in text :
|
||||
val = float(text.split(' ')[0])
|
||||
else :
|
||||
val = None
|
||||
return val
|
||||
|
||||
def parse_artcraft_price (html) :
|
||||
# with open('./test.html', 'w') as file :
|
||||
# file.write(html)
|
||||
d = pq(html)
|
||||
price = None
|
||||
for p in d('span.price-item').items() :
|
||||
price = p.text()
|
||||
break
|
||||
#print(price)
|
||||
if price is None :
|
||||
return None
|
||||
dollars = price.replace('$', '').replace(' ', '').replace('USD', '').strip()
|
||||
#print(dollars)
|
||||
cents = math.ceil(float(dollars) * 100.0)
|
||||
return cents
|
||||
|
||||
def parse_artcraft_product (html) :
|
||||
d = pq(html)
|
||||
data = []
|
||||
scr = d('variant-radios script').text()
|
||||
objs = json.loads(scr)
|
||||
for o in objs:
|
||||
data.append({
|
||||
'id' : o['id'],
|
||||
'weight' : parse_artcraft_weight(o['title'])
|
||||
})
|
||||
return data
|
||||
|
||||
async def main () :
|
||||
parser = ArgumentParser(description='Refresh prices from sources')
|
||||
parser.add_argument('-s', '--source', type=is_source, required=False, default=None, help='Only run on single source')
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(csvOutput, 'w') as file :
|
||||
file.write('chemical,url,grams,milliliters,price' + EOL)
|
||||
|
||||
with open(csvFile, newline='') as csvfile:
|
||||
reader = csv.reader(csvfile, delimiter=',', quotechar='|')
|
||||
for row in reader:
|
||||
source = row[0]
|
||||
chemical = row[1]
|
||||
url = row[2]
|
||||
if source == 'source' or (args.source is not None and source != args.source) :
|
||||
continue
|
||||
if source == 'artcraft' :
|
||||
await artcraft(url, chemical)
|
||||
|
||||
if __name__ == '__main__' :
|
||||
asyncio.get_event_loop().run_until_complete(main())
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
pyppeteer
|
||||
pyquery
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
source,chemical,url
|
||||
artcraft,Sodium Sulfite (anhydrous),https://artcraftchemicals.com/products/sodium-sulfite-anhy-part-1416
|
||||
artcraft,Sodium Carbonate (monohydrate),https://artcraftchemicals.com/products/sodium-carbonate-mono-part-1340
|
||||
artcraft,Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234
|
||||
artcraft,Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200
|
||||
artcraft,Metol,https://artcraftchemicals.com/products/metol-part-1185
|
||||
artcraft,Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180
|
||||
artcraft,Sodium Metabisulfite,https://artcraftchemicals.com/products/sodium-metabisulfite-part-1375
|
||||
artcraft,Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435
|
||||
|
37
supply.csv
37
supply.csv
|
|
@ -1,15 +1,24 @@
|
|||
chemical,url,grams,milliliters,price
|
||||
Sodium Sulfite (anhydrous),https://www.bhphotovideo.com/c/product/124092-REG/Photographers_Formulary_10_1340_1LB_Sodium_Sulfite_1.html,453.592,,695
|
||||
Sodium Carbonate (monohydrate),https://www.bhphotovideo.com/c/product/124044-REG/Photographers_Formulary_10_1190_1LB_Sodium_Carbonate_Monohydrate.html,453.592,,995
|
||||
Potassium Bromide,https://www.bhphotovideo.com/c/product/123938-REG/Photographers_Formulary_10_0930_1LB_Potassium_Bromide_1.html,453.592,,2195
|
||||
Phenidone,https://www.bhphotovideo.com/c/product/123915-REG/Photographers_Formulary_10_0870_100G_Phenidone_100_Grams.html,100,,2695
|
||||
Metol,https://www.bhphotovideo.com/c/product/123903-REG/Photographers_Formulary_10_0770_100G_Metol_Elon_100.html,100,,1695
|
||||
Hydroquinone,https://www.bhphotovideo.com/c/product/123903-REG/Photographers_Formulary_10_0770_100G_Metol_Elon_100.html,100,,1695
|
||||
Glycin,https://www.bhphotovideo.com/c/product/123868-REG/Photographers_Formulary_10_0610_100G_Glycin_N_Parahydroxyphenyl.html,100,,2395
|
||||
Borax,https://www.bhphotovideo.com/c/product/123755-REG/Photographers_Formulary_10_0260_1LB_Borax_1_lb.html,453.592,,695
|
||||
Hydroquinone,https://www.bhphotovideo.com/c/product/123887-REG/Photographers_Formulary_10_0670_100G_Hydroquinone_100_Grams.html,100,,895
|
||||
Sodium Metabisulfite,https://www.bhphotovideo.com/c/product/124063-REG/Photographers_Formulary_10_1280_100G_Sodium_Metabisulfite_100g.html,100,,595
|
||||
Sodium Metaborate,https://www.bhphotovideo.com/c/product/124068-REG/Photographers_Formulary_10_1285_1LB_Sodium_Metaborate_Balanced_Alkali.html,453.592,,1195
|
||||
Potassium Metabisulfite,https://morewinemaking.com/products/potassium-metabisulfite.html,453.592,,1199
|
||||
Sodium Hydroxide,https://www.amazon.com/Sodium-Hydroxide-Grade-Caustic-Pound/dp/B07KNR9SVF,907.184,,1799
|
||||
p-Aminophenol Hydrochloride,https://www.sigmaaldrich.com/US/en/product/sial/09170,100,,10300
|
||||
Sodium Sulfite (anhydrous),https://artcraftchemicals.com/products/sodium-sulfite-anhy-part-1416?variant=42892710904055,2267.96,,1811
|
||||
Sodium Sulfite (anhydrous),https://artcraftchemicals.com/products/sodium-sulfite-anhy-part-1416?variant=42892710969591,9071.84,,5457
|
||||
Sodium Sulfite (anhydrous),https://artcraftchemicals.com/products/sodium-sulfite-anhy-part-1416?variant=42892711035127,453.592,,663
|
||||
Sodium Carbonate (monohydrate),https://artcraftchemicals.com/products/sodium-carbonate-mono-part-1340?variant=42892721979639,453.592,,785
|
||||
Sodium Carbonate (monohydrate),https://artcraftchemicals.com/products/sodium-carbonate-mono-part-1340?variant=42892722012407,2041.164,,2922
|
||||
Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234?variant=42892792266999,100.0,,791
|
||||
Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234?variant=42892792299767,250.0,,1265
|
||||
Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234?variant=42892792332535,453.592,,2036
|
||||
Potassium Bromide,https://artcraftchemicals.com/products/potassium-bromide-part-1234?variant=43227267006711,2267.96,,8058
|
||||
Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200?variant=42892799639799,50.0,,1474
|
||||
Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200?variant=42892799672567,100.0,,2428
|
||||
Phenidone,https://artcraftchemicals.com/products/phenidone-a-part-1200?variant=42892799705335,250.0,,5319
|
||||
Metol,https://artcraftchemicals.com/products/metol-part-1185?variant=42892802851063,100.0,,1831
|
||||
Metol,https://artcraftchemicals.com/products/metol-part-1185?variant=42892802883831,250.0,,3973
|
||||
Metol,https://artcraftchemicals.com/products/metol-part-1185?variant=42892802916599,453.592,,6217
|
||||
Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180?variant=42892822675703,453.592,,2346
|
||||
Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180?variant=42892822708471,250.0,,1479
|
||||
Hydroquinone,https://artcraftchemicals.com/products/hydroquinone-1-lb-part-1180?variant=43085712654583,2267.96,,9410
|
||||
Sodium Metabisulfite,https://artcraftchemicals.com/products/sodium-metabisulfite-part-1375?variant=42892717326583,453.592,,729
|
||||
Sodium Metabisulfite,https://artcraftchemicals.com/products/sodium-metabisulfite-part-1375?variant=42892717359351,2267.96,,2474
|
||||
Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435?variant=42892693602551,453.592,,709
|
||||
Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435?variant=42892693635319,2267.96,,2036
|
||||
Sodium Thiosulfate,https://artcraftchemicals.com/products/sodium-thiosulfate-penta-part-1435?variant=42892693668087,8618.248,,5809
|
||||
|
|
|
|||
|
|
|
@ -0,0 +1,18 @@
|
|||
chemical,url,grams,milliliters,price
|
||||
Sodium Sulfite (anhydrous),https://www.bhphotovideo.com/c/product/124092-REG/Photographers_Formulary_10_1340_1LB_Sodium_Sulfite_1.html,453.592,,695
|
||||
Sodium Carbonate (monohydrate),https://www.bhphotovideo.com/c/product/124044-REG/Photographers_Formulary_10_1190_1LB_Sodium_Carbonate_Monohydrate.html,453.592,,995
|
||||
Potassium Bromide,https://www.bhphotovideo.com/c/product/123938-REG/Photographers_Formulary_10_0930_1LB_Potassium_Bromide_1.html,453.592,,2195
|
||||
Phenidone,https://www.bhphotovideo.com/c/product/123915-REG/Photographers_Formulary_10_0870_100G_Phenidone_100_Grams.html,100,,2695
|
||||
Metol,https://www.bhphotovideo.com/c/product/123903-REG/Photographers_Formulary_10_0770_100G_Metol_Elon_100.html,100,,1695
|
||||
Hydroquinone,https://www.bhphotovideo.com/c/product/123903-REG/Photographers_Formulary_10_0770_100G_Metol_Elon_100.html,100,,1695
|
||||
Glycin,https://www.bhphotovideo.com/c/product/123868-REG/Photographers_Formulary_10_0610_100G_Glycin_N_Parahydroxyphenyl.html,100,,2395
|
||||
Borax,https://www.bhphotovideo.com/c/product/123755-REG/Photographers_Formulary_10_0260_1LB_Borax_1_lb.html,453.592,,695
|
||||
Borax,https://artcraftchemicals.com/products/borax-part-1070,453.592,,663
|
||||
Hydroquinone,https://www.bhphotovideo.com/c/product/123887-REG/Photographers_Formulary_10_0670_100G_Hydroquinone_100_Grams.html,100,,895
|
||||
Sodium Metabisulfite,https://www.bhphotovideo.com/c/product/124063-REG/Photographers_Formulary_10_1280_100G_Sodium_Metabisulfite_100g.html,100,,595
|
||||
Sodium Metaborate,https://www.bhphotovideo.com/c/product/124068-REG/Photographers_Formulary_10_1285_1LB_Sodium_Metaborate_Balanced_Alkali.html,453.592,,1195
|
||||
Sodium Metaborate,https://artcraftchemicals.com/products/sodium-metaborate-bal-alkali-part-1385,453.592,,1015
|
||||
Potassium Metabisulfite,https://morewinemaking.com/products/potassium-metabisulfite.html,453.592,,1199
|
||||
Sodium Hydroxide,https://www.amazon.com/Sodium-Hydroxide-Grade-Caustic-Pound/dp/B07KNR9SVF,907.184,,1799
|
||||
Sodium Hydroxide,https://artcraftchemicals.com/products/sodium-hydroxide-part-1759,453.592,,1015
|
||||
p-Aminophenol Hydrochloride,https://www.sigmaaldrich.com/US/en/product/sial/09170,100,,10300
|
||||
|
Loading…
Reference in New Issue