var express = require('express'); var router = express.Router(); var firebase = require('firebase'); var request = require('request'); const cheerio = require('cheerio'); const axios = require('axios'); const schedule = require('node-schedule'); const serverUrlBase = "https://safe-beach-33580.herokuapp.com"; const clothesRef = firebase.database().ref('ubrania'); const newDropRef = firebase.database().ref('isNewDrop'); const supremeHtmlRef = firebase.database().ref('supremeHtml'); var clothesObj = []; var isNewDrop = false; newDropRef.set(isNewDrop); //przypisuje zmiennej firstItemAtSupremeShop wartość z firebase po starcie serwera. supremeHtmlRef.on("value", function(snapshot) { console.log('html ze sklepu pobrany z firebase: ', snapshot.val()); firstItemAtSupremeShop = snapshot.val(); }, function (errorObject) { console.log("The read failed: " + errorObject.code); }); router.get('/checkfordrop', function (req, res, next) { checkForNewDrop(); }); router.get('/isnewdrop', function (req, res, next) { newDropRef.set(isNewDrop); res.send(isNewDrop); console.log("isNewDrop=" + isNewDrop); }); router.get('/setisnewdroptofalse', function (req, res, next) { isNewDrop = false; newDropRef.set(isNewDrop); }); router.get('/clothes', function (req, res, next) { clothesRef.once('value', function (snapshot) { res.json(snapshot.val()); }); }); router.get('/clothes/update', function (req, res, next) { request('https://www.supremecommunity.com/season/spring-summer2018/droplists/', function (error, response, html) { if (!error && response.statusCode == 200) { const $ = cheerio.load(html); const url_after_slash = $('.feature.feature-7.boxed.text-center.imagebg.boxedred').parent().attr('href'); var url = 'https://www.supremecommunity.com' + url_after_slash; console.log(url); request(url, function (error, response, html) { var newData = []; const $ = cheerio.load(html); $('.card-details').each(function (i) { const card = $(this); const name = card.find('.name.item-details').text(); const url = 'https://www.supremecommunity.com' + card.find('img').attr('src'); const price = card.find('span.label-price').text(); var category = card.parent().parent().attr('data-masonry-filter'); category = repairCategoryName(category); var sizeArray = chooseSizeArrayByCategory(category); sizeArray = getSpecialSizeArray(name, sizeArray); const colorArray = getSpecialColorArray(name); const nameRegex = getNameRegex(name); const colorRegexArray = getColorRegexArray(name); newData.push({ name: name, url: url, price: price, category: category, sizeArray: sizeArray, colorArray: colorArray, nameRegex: nameRegex, colorRegexArray: colorRegexArray }); }); //dodaje testowe itemy for (var key in itemsForTest) { if (itemsForTest.hasOwnProperty(key)) { const name = key; console.log("testItem name: ", name); const url = "http://icons.iconarchive.com/icons/pelfusion/flat-file-type/512/jpg-icon.png"; const price = "bezcenne"; var category = itemsForTest[key].category; category = repairCategoryName(category); var sizeArray = chooseSizeArrayByCategory(category); sizeArray = getSpecialSizeArray(name, sizeArray); const colorArray = getSpecialColorArray(name); const nameRegex = getNameRegex(name); const colorRegexArray = getColorRegexArray(name); newData.push({ name: name, url: url, price: price, category: category, sizeArray: sizeArray, colorArray: colorArray, nameRegex: nameRegex, colorRegexArray: colorRegexArray }); } } console.log(newData); clothesObj = newData; clothesRef.set(newData); res.sendStatus(200); }); } }); }); function chooseSizeArrayByCategory(category) { var sizeArrays = { jackets: ['Small', 'Medium', 'Large', 'XLarge'], shirts: ['Small', 'Medium', 'Large', 'XLarge'], tops_sweaters: ['Small', 'Medium', 'Large', 'XLarge'], 't-shirts': ['Small', 'Medium', 'Large', 'XLarge'], sweatshirts: ['Small', 'Medium', 'Large', 'XLarge'], pants: ['30', '32', '34', '36'], shoes: ['38', '39', '40', '41', '42', '43', '44', '45', '46' ], hats: ['7 1/8', '7 1/4', '7 3/8', '7 1/2', '7 5/8', '7 3/4'], accessories: ['Small', 'Medium', 'Large', 'XLarge'], bags: null, skate: null, shorts: ['Small', 'Medium', 'Large', 'XLarge'], }; return sizeArrays[category]; } var itemsForTest = { "Supreme®/Hanes® Tagle Tees (3 Pack)": {category: "accessories"}, "Supreme®/Hanes® Cr Socks (4 Pack)": {category: "accessories"}, "Studded Logo Belt": {category: "accessories"} }; var specialSizeAndColor = { 'Supreme®/LACOSTE Velour Bucket' : {colorArray: ['Black'], colorRegexArray: ['.*B.*l.*a.*c.*k.*'], nameRegex: '.*S.*u.*p.*r.*e.*m.*e.*L.*A.*C.*O.*S.*T.*E.*V.*e.*l.*o.*u.*r.*B.*u.*c.*k.*e.*t.*'}, 'Supreme®/LACOSTE Crewneck Sweatshirt' : {colorArray: ['Black'], colorRegexArray: ['.*B.*l.*a.*c.*k.*'], nameRegex: '.*S.*u.*p.*r.*e.*m.*e.*L.*A.*C.*O.*S.*T.*E.*C.*r.*e.*w.*n.*e.*c.*k.*S.*w.*e.*a.*t.*s.*h.*i.*r.*t.*'}, 'Supreme®/LACOSTE Hooded Sweatshirt' : {colorArray: ['Black'], colorRegexArray: ['.*B.*l.*a.*c.*k.*'], nameRegex: '.*S.*u.*p.*r.*e.*m.*e.*L.*A.*C.*O.*S.*T.*E.*H.*o.*o.*d.*e.*d.*S.*w.*e.*a.*t.*s.*h.*i.*r.*t.*'}, 'Sideline Hooded Sweatshirt' : {colorArray: ['Black'], colorRegexArray: ['.*B.*l.*a.*c.*k.*'], nameRegex: '.*S.*i.*d.*e.*l.*i.*n.*e.*H.*o.*o.*d.*e.*d.*S.*w.*e.*a.*t.*s.*h.*i.*r.*t.*'}, /* 'Supreme®/Wilson® Tennis Balls': {colorArray: null, colorRegexArray: null, nameRegex: '.*S.*u.*p.*r.*e.*m.*e.*W.*i.*l.*s.*o.*n.* T.*e.*n.*n.*i.*s.* B.*a.*l.*l.*s.*', sizeArray: null }, 'Supreme®/UNDERCOVER/Public Enemy White House Tee' : {colorArray: ['Black'], colorRegexArray: ['.*B.*l.*a.*c.*k.*'], nameRegex: '.*S.*u.*p.*r.*e.*m.*e.*U.*N.*D.*E.*R.*C.*O.*V.*E.*R.*P.*u.*b.*l.*i.*c.*E.*n.*e.*m.*y.*W.*h.*i.*t.*e.*H.*o.*u.*s.*e.*T.*e.*e.*'}, 'Supreme®/SIGG™ Traveller 0.6L Water Bottle': {colorArray: null, colorRegexArray: null, nameRegex: '.*S.*u.*p.*r.*e.*m.*e.*S.*I.*G.*G.*T.*r.*a.*v.*e.*l.*l.*e.*r.*0.*..*6.*L.*W.*a.*t.*e.*r.*B.*o.*t.*t.*l.*e.*', sizeArray: null }, 'FTW Tee': {colorArray: ['Black'], colorRegexArray: ['.*B.*l.*a.*c.*k.*'], nameRegex: '.*F.*T.*W.*T.*e.*e.*' }, 'Logo Zippo®': {colorArray: ['Red'], colorRegexArray: ['Red'], nameRegex: '.*Zippo.*', sizeArray: null }, 'Studded Logo Belt': {colorArray: ['Red'], colorRegexArray: ['.*R.*e.*d.*'], nameRegex: '.*S.*t.*u.*d.*d.*e.*d.*L.*o.*g.*o.*B.*e.*l.*t.*', sizeArray: null }, 'Illegal Business Hooded Sweatshirt': {colorArray: ['Brown', 'Black', 'Pale Pink', 'White', 'Blue', 'Red', 'Pale Yellow'], colorRegexArray: ['.*Brown', 'Black', '.*Pink', 'White', '(.*Blue)|Royal', 'Red', '.*(Yellow|Lime)'], nameRegex: 'Illegal Business Hooded Sweatshirt' }, 'Supreme®/Stone Island® Hooded Sweatshirt': {colorArray: ['Black', 'Orange', 'Red', 'Khaki', 'White', 'Orange', 'Grey'], colorRegexArray: ['Black', '.*Orange.*', 'Red', 'Khaki|Olive', '.*(White|Grey)'], nameRegex: 'Supreme..Stone Island. Hooded Sweatshirt' }, 'Supreme®/Stone Island® Poly Cover Composite Anorak': {colorArray: ['Black', 'Navy', 'Yellow'], colorRegexArray: ['(Black|((Dark)? Grey))', '(Navy|((Dark)? Blue)', '.*(Yellow|Lime|Marigold)'], nameRegex: 'Supreme..Stone Island. (Poly)? (Cover)? Composite Anorak'}, 'Supreme®/Stone Island® S/S Top': {colorArray: ['Black', 'Khaki', 'Red', 'Orange', 'White'], colorRegexArray: ['Black', 'Khaki|Olive', 'Red', 'Orange', 'White'], nameRegex: 'Supreme..Stone Island. S.S Top'}, 'Supreme®/Stone Island® Sweatpant': {colorArray: ['Black', 'Orange', 'Red', 'Khaki', 'White'], colorRegexArray: ['Black', 'Khaki|Olive', 'Red', 'Orange', 'White'], nameRegex: 'Supreme..Stone Island. Sweatpant.*' }, 'Supreme®/Stone Island® Lamy Cover Stampato Puffy Jacket': {colorArray: ['Black', 'White'], colorRegexArray: ['Black', '(White|Ecru)'], nameRegex: 'Supreme..Stone Island. Lamy Cover Stampato Puffy Jacket'}, 'Lock Box': {colorArray: null, sizeArray: null, nameRegex: '.* Lock Box'}, 'Supreme®/Stone Island® Lamy 6-Panel': {colorArray: ['Black', 'White'], sizeArray: null, colorRegexArray: ['Black', '(White|Ecru)'], nameRegex: 'Supreme..Stone Island. Lamy 6.Panel' }, 'Heather Loose Gauge Beanie': {colorArray: ['Burgundy', 'Black', 'Brown', 'Navy', 'Green', 'Purple', 'Salmon'], sizeArray: null, colorRegexArray: [".*(Burgundy|Red)", 'Black', '.*(Brown|Mustard)', 'Navy', 'Green|Teal', 'Purple|Plum', '.*(Pink|Salmon|Peach|Orange)']}, 'Arc Logo L/S Thermal': {colorArray: ['Black', 'White', 'Red', 'Purple', 'Navy', 'Yellow', 'Grey', 'Camo'], colorRegexArray: ['Black', 'White', 'Red', '.*(Purple|Plum)', 'Navy', '.*Yellow', '.*Grey', '.*Camo'] }, 'Side Zip Camp Cap': {colorArray: ['Navy', 'Lime', 'Red', 'Blue', 'Camo', 'Black', 'Pink'], colorRegexArray: ['Navy', '.*(White|Lime)', 'Red', '.*Blue', '.*Camo', 'Black', '.*Pink'] }, 'Shop Jacket': {colorArray: ['Red', 'Black', 'Khaki', 'White', 'Red', 'Navy'] }, 'Split Pique Crewneck': {colorArray: ['Black', 'Khaki', 'Pink'], colorRegexArray: ['Black', '.*(Khaki|Olive)', '.*Pink'] }, 'Daggers L/S Top': {colorArray: ['White', 'Pink', 'Khaki'], colorRegexArray: ['White', '.*(Pink|Salmon)', 'Khaki'] }, 'Daggers Pant': {colorArray: ['White', 'Pink', 'Khaki'], colorRegexArray: ['White', '.*(Pink|Salmon)', 'Khaki'] }, 'God Bless Plaid Flannel Shirt': {colorArray: ['Yellow', 'Pink', 'Black', 'Blue'], colorRegexArray: ['Yellow', '.*(Pink|Salmon)', 'Black', 'Blue'] }, 'Repeat Beanie': {colorArray: ['Black', 'Red', 'White'] }, 'God Bless Plaid Flannel Shirt': {colorArray: ['Yellow', 'Pink', 'Black', 'Blue'] }, 'Corduroy Work Pant': {colorArray: ['Black', 'White', 'Brown', 'Navy', 'Salmon', 'Red'] }, 'Leaf Beanie': {colorArray: ['Pink', 'Black', 'White', 'Blue', 'Red'] }, "Reflective Excellence Hooded Sweatshirt": {colorArray: ['Green', 'Navy', 'Red', 'White', 'Pink', 'Blue', 'Black'], colorRegexArray: ['.*Green', 'Navy', 'Red', 'White', '.*Pink', "Blue|Royal", 'Black']} */ "Supreme®/Hanes® Cr Socks (4 Pack)": {nameRegex: 'Supreme®.Hanes® Cr.. Socks .4 Pack.'} }; function getSpecialSizeArray(name, sizeArray) { //console.log(specialSizeAndColor[name]); if (specialSizeAndColor[name] !== undefined && specialSizeAndColor[name].sizeArray !== undefined) { return specialSizeAndColor[name].sizeArray; } return sizeArray; } function getSpecialColorArray(name) { if (specialSizeAndColor[name] !== undefined && specialSizeAndColor[name].colorArray !== undefined) { return specialSizeAndColor[name].colorArray; } return null; } function getNameRegex(name) { if (specialSizeAndColor[name] !== undefined && specialSizeAndColor[name].nameRegex !== undefined) { return specialSizeAndColor[name].nameRegex; } return null; } function getColorRegexArray(name) { if (specialSizeAndColor[name] !== undefined && specialSizeAndColor[name].colorRegexArray !== undefined) { return specialSizeAndColor[name].colorRegexArray; } return null; } function repairCategoryName(categoryName) { if (categoryName === 'tops-sweaters') { categoryName = 'tops_sweaters'; } return categoryName; } /* Sprawdza zaraz przed dropem czy pojawiły się już nowe itemy na stronie supreme. Porównuje pierwszy item z itemem, który jest w bazie danych jako "pierwszy z brzegu" */ const checkForUpdatesBeforeDrop = schedule.scheduleJob('0 58 10 * * 4', function(){ checkForNewDrop(); }); /* Co minute sprawdza czy na supreme comunnity są jakieś nowe itemy. */ const checkForUpdatesEveryMinute = schedule.scheduleJob('*/1 * * * *', async function(){ console.log('every minute update'); axios.get(serverUrlBase + '/clothes/update') .then(function (response) { console.log("every minute update response (probably OK)"); }) .catch(function (error) { console.log("ERRROR"); console.log(error); }); }); function makeRegex(name){ name = name.replace(/\s/g, ''); name = name.replace(/[\W_]+/g, ''); name = name.split('').join('.*'); name = ".*" + name; name = name + ".*"; return name; } async function checkForNewDrop () { var newItemCheckCounter = 0; var isUpdated = false; while(!isUpdated) { console.log("while loop, counter: " + newItemCheckCounter); console.log("clothes, counter: " + clothesObj.length); isUpdated = await getNewDropItem(newItemCheckCounter); newItemCheckCounter++; newItemCheckCounter=newItemCheckCounter%clothesObj.length; } isNewDrop = true; newDropRef.set(isNewDrop); } function getNewDropItem(i) { console.log("function getNewDropItem()"); var itemToSearch = { name: clothesObj[i].name, category: clothesObj[i].category, nameRegexString: makeRegex(clothesObj[i].name), nameRegexObj: new RegExp(this.nameRegexString) }; console.log("function getNewDropItem()"); console.log("Name of item we're looking for: "+itemToSearch.name); console.log("nameRegexString: "+itemToSearch.nameRegexString); return new Promise(function(resolve, reject) { console.log("promise"); const url = 'http://www.supremenewyork.com/shop/all/'+itemToSearch.category; console.log("Name of category page: "+ url); request(url, {timeout: 2000}, function (error, response, html) { console.log("request"); console.log(error); if (error){ console.log('is timeout error: ', error.code === 'ETIMEDOUT'); console.log('is timeout connection error: ', error.connect === 'true'); } if (!error && response.statusCode == 200) { console.log("OK"); const $ = cheerio.load(html); var isItemFound = itemToSearch.nameRegexObj.test($); console.log("isItemFound: "+isItemFound); resolve(isItemFound); } resolve(false); }); }); } module.exports = router;