var async = require('async'); var request = require('request'); var cheerio = require('cheerio'); var User = require('../app/models/user'); var csv = require('./csv'); require('../config/vars'); function getPrice(priceStr){ var priceRg = /(?:-|\.|,|\d)*$/ig; var res = priceRg.exec(priceStr); var price = parseFloat(res[0]); price = price?price:0; return price; } exports.scrapAmazonItems = function(req, checkboxes, ebayData, callback) { // Get user settings var paypalEmail = ""; var postcode = ""; var ebayCountry = ""; User.findOne({ _id: req.user._id }, function(err, user) { if (!err) { paypalEmail = user.local.settings.paypalEmail; postcode = user.local.settings.zip; ebayCountry = user.local.settings.ebayCountry; } }); var list = []; async.forEach(checkboxes, function(item, callback) { //The second argument (callback) is the "task callback" for a specific messageId // If item (URL) is undefined, ignore this loop if (typeof item === undefined) { callback(); } var url = item; // var newURL = url.replace(/\/dp\//, "/dp/a/"); var newURL = url; console.log("url", newURL); var newEbay = JSON.parse(ebayData[item]); // images //var re = /"large":"([^"]*)"/g; var testRe = /http:\/\/ecx\.images\-amazon\.com\/images\/I/ig; var reHiRes = /"hiRes":"(.*?)"|"hiRes":()null/ig; var reLarge = /"large":"(.*?)"|"large":()null/ig; // var re = /
<\/div>/ig; // end images request(newURL, function(err, response, html) { console.log("newURL", newURL); if (err) { console.log(err) } else { var $ = cheerio.load(html); // images var imgs = []; /*while(true){ var res = re.exec(html); if(!res){ break; } if(res[1]){ var img = res[1].replace(/\.[^\.]*?\.jpg/i,"._SL500_.jpg"); imgs.push(img); } } console.log("----- Images ----: "); console.log(imgs);*/ $('script').each(function(index, el) { var js = $(this).text(); if(testRe.test(js)){ // find hires images var hiresImgs = []; while(true){ var res = reHiRes.exec(js); console.log(res); if(!res){ break; } hiresImgs.push(res[1] || res[2]); } // find large images var largeImgs = []; while(true){ var res = reLarge.exec(js); if(!res){ break; } largeImgs.push(res[1] || res[2]); } // merge image, hires higher priority for(var i=0;i 0.1)) { //console.log("item", item); newList.push(item); } } console.log("calling callback " + i) callback(); console.log("here"); }); }, function(err) { //This is the final callback if (err) return callback(err, null); console.log("entering final callback"); // filter list var resultList = []; for(var i=0;i profitTo){ continue; } resultList.push(newList[i]); } console.log("calling final callback") callback(err, resultList); }); } function getAmazonURL(country) { var url = ''; switch (country) { case 'at': url = 'http://www.amazon.at/'; break; case 'ca': url = 'http://www.amazon.ca/'; break; case 'fr': url = 'http://www.amazon.fr/'; break; case 'it': url = 'http://www.amazon.it/'; break; case 'es': url = 'http://www.amazon.es/'; break; case 'de': url = 'http://www.amazon.de/'; break; case 'us': url = 'http://www.amazon.com/'; break; default: url = 'http://www.amazon.co.uk/'; break; } return url; }