var async = require('async');
var request = require('request');
var cheerio = require('cheerio');
var User = require('../app/models/user');
var csv = require('./csv');
require('../config/vars');
function getPrice(priceStr){
var priceRg = /(?:-|\.|,|\d)*$/ig;
var res = priceRg.exec(priceStr);
var price = parseFloat(res[0]);
price = price?price:0;
return price;
}
exports.scrapAmazonItems = function(req, checkboxes, ebayData, callback) {
// Get user settings
var paypalEmail = "";
var postcode = "";
var ebayCountry = "";
User.findOne({
_id: req.user._id
}, function(err, user) {
if (!err) {
paypalEmail = user.local.settings.paypalEmail;
postcode = user.local.settings.zip;
ebayCountry = user.local.settings.ebayCountry;
}
});
var list = [];
async.forEach(checkboxes, function(item, callback) { //The second argument (callback) is the "task callback" for a specific messageId
// If item (URL) is undefined, ignore this loop
if (typeof item === undefined) {
callback();
}
var url = item;
// var newURL = url.replace(/\/dp\//, "/dp/a/");
var newURL = url;
console.log("url", newURL);
var newEbay = JSON.parse(ebayData[item]);
// images
//var re = /"large":"([^"]*)"/g;
var testRe = /http:\/\/ecx\.images\-amazon\.com\/images\/I/ig;
var reHiRes = /"hiRes":"(.*?)"|"hiRes":()null/ig;
var reLarge = /"large":"(.*?)"|"large":()null/ig;
// var re = /<div id="thumb_image_.*?" class="thumb.*?"><img alt="" src="(http:\/\/ecx\.images-amazon\.com\/images\/I\/.*?)"><\/div>/ig;
// end images
request(newURL, function(err, response, html) {
console.log("newURL", newURL);
if (err) {
console.log(err)
} else {
var $ = cheerio.load(html);
// images
var imgs = [];
/*while(true){
var res = re.exec(html);
if(!res){
break;
}
if(res[1]){
var img = res[1].replace(/\.[^\.]*?\.jpg/i,"._SL500_.jpg");
imgs.push(img);
}
}
console.log("----- Images ----: ");
console.log(imgs);*/
$('script').each(function(index, el) {
var js = $(this).text();
if(testRe.test(js)){
// find hires images
var hiresImgs = [];
while(true){
var res = reHiRes.exec(js);
console.log(res);
if(!res){
break;
}
hiresImgs.push(res[1] || res[2]);
}
// find large images
var largeImgs = [];
while(true){
var res = reLarge.exec(js);
if(!res){
break;
}
largeImgs.push(res[1] || res[2]);
}
// merge image, hires higher priority
for(var i=0;i<hiresImgs.length;i++){
if(hiresImgs[i]){
imgs.push(hiresImgs[i]);
}else if(largeImgs[i]){
imgs.push(largeImgs[i]);
}
}
return ;
}
});
console.log("images", imgs);
// end images
// Remove images
$('.leftImage').remove();
$('.rightImage').remove();
// Remove seeAll link
$('div.seeAll').remove();
// Remove duplicate product description
$('h3.productDescriptionSource').remove();
var d = $('#btAsinTitle').children().text();
var l = $('.priceLarge').text();
var j = {};
$('#thumbs-image a').each(function() {
j[$(this).children().attr('src')] = []
});
var p = $('#technicalProductFeaturesATF').children('ul').html();
var s = $('#productDescription').html();
if (!d) {
d = $("#productTitle").text();
l = $("#priceblock_ourprice").text();
// j = JSON.parse($("#imgTagWrapperId").children().attr('data-a-dynamic-image'));
j = imgs[0];
p = $('#feature-bullets').children('ul').html() ? $('#feature-bullets').children('ul').html() : $('#feature-bullets').children().children().html();
//s = $(".techD").html();
// get product description
var iframeRg = /var iframeContent = "(.*?)";/ig;
var iframeRs = iframeRg.exec(html);
if(iframeRs && iframeRs[1]){
var iframeContent = decodeURIComponent(iframeRs[1]);
var $desc = cheerio.load(iframeContent);
// Remove seeAll link
$desc('div.seeAll').remove();
// Remove duplicate product description
$desc('h3.productDescriptionSource').remove();
//s = $desc('#productDescription').html();
s = $desc('#productDescription div.productDescriptionWrapper').html();
}else{
s = $('#productDescription div.productDescriptionWrapper').html();
}
}
s.replace(/(\r\n|\n|\r)/gm,"");
// Replace non-alpha numeric characters from title with a space
d = d.replace(/([^0-9A-Za-z\.\s]+)/g, '').replace(/(\s\s)/, '').trim();
p = p ? '"' + p.replace(/\n/g, '').replace(/\t/g, ' ') + '"' : '';
l = l.replace('�', '£');
// Replace new lines
s = s ? s.replace(/\n/g, '') : '';
var countries = req.app.locals.ebayCountries;
var ebaySiteID = '';
//console.log("countries", countries);
for (index in countries) {
///console.log("ebayCountry", ebayCountry);
if (countries[index].value === ebayCountry) {
//console.log("match found - " + countries[index].ebaySiteID);
ebaySiteID = countries[index].ebaySiteID;
ebayCurrency = countries[index].ebayCurrency;
ebayCountryText = countries[index].ebayCountry;
}
//console.log(countries[index].label);
}
var n = {
d: d,
l: l,
j: j,
p: p,
s: s,
url: url,
postcode: postcode,
paypalEmail: paypalEmail,
ebaySiteID: ebaySiteID,
ebayCurrency: ebayCurrency,
ebayPrice: newEbay.eprice
};
list.push(n);
}
callback();
});
}, function(err) { //This is the final callback
csv.createCSV(list, callback);
});
}
exports.scrapAmazonList = function(req, list, options, callback) {
var profitFrom = parseFloat(req.body.profitFrom);
var profitTo = parseFloat(req.body.profitTo);
// Get user settings
var paypalFees = 0;
var eBayFees = 0;
User.findOne({
_id: req.user._id
}, function(err, user) {
//console.log("user findone");
if (!err) {
paypalFees = user.local.settings.paypalFee;
eBayFees = user.local.settings.ebayListingFee;
}
});
var i = 0;
var newList = [];
list = list.slice(0,20); // for fast test
/* console.log("list.length");
console.log(list.length);*/
console.log("list length", list.length)
async.forEach(list, function(item, callback) { //The second argument (callback) is the "task callback" for a specific messageId
var amazonURL = getAmazonURL(req.session.ebayCountry);
var url = amazonURL + 's/ref=nb_sb_noss?url=search-alias%3Daps&field-keywords=' + item.text;
i++;
request(url, function(err, response, html) {
if (err) {
console.log("error")
callback(err);
} else {
console.log("else");
var $ = cheerio.load(html);
item.amazonTitle = $("#result_0 .bold").html();
item.amazonPrice = $("#result_0 .red").html();
item.aprice = item.amazonPrice ? parseFloat(item.amazonPrice.replace(/^\D+/g, '')) : 0;
item.eprice = item.price ? parseFloat(item.price.replace(/^\D+/g, '')) : 0;
item.aurl = $("#result_0 .newaps").children().attr('href');
var paypalFeesAsPercentage = (parseFloat(paypalFees) / 100);
//console.log("price", item.price);
//console.log("eprice", item.eprice);
item.paypalFees = Math.round((parseFloat(item.eprice) * paypalFeesAsPercentage) * 100) / 100;
item.ebayFees = Math.round(((parseFloat(item.eprice) / 10) + parseFloat(eBayFees)) * 100) / 100;
// item.profit1 = Math.round(((parseFloat(item.eprice) - (item.paypalFees + item.ebayFees)) - parseFloat(item.aprice)) * 100) / 100;
/*console.log("---------------- 1111111 --------------");
console.log(item.eprice);
console.log(item.paypalFees);
console.log(item.ebayFees);
console.log(item.aprice);
console.log("---------------- 2222222 --------------");
console.log(parseFloat(item.eprice));
console.log(item.paypalFees + item.ebayFees);
console.log(getPrice(item.aprice));*/
item.profit = Math.round(((parseFloat(item.eprice) - (item.paypalFees + item.ebayFees)) - getPrice(item.aprice)) * 100) / 100;
var currencySymbol = '';
var countries = req.app.locals.ebayCountries;
for(var i=0;i<countries.length;i++){
if(countries[i].value == req.body.ebayCountry){
currencySymbol = countries[i]['currencySymbol'];
}
}
item.paypalFees = currencySymbol + item.paypalFees.toFixed(2);
item.ebayFees = currencySymbol + item.ebayFees.toFixed(2);
var profit = item.profit.toFixed(2);
item.profit = currencySymbol + item.profit.toFixed(2);
item.ebayData = JSON.stringify(item);
// If searching for all items OR
if (options.listingsType == "all" || (options.listingsType != "all" && profit > 0.1)) {
//console.log("item", item);
newList.push(item);
}
}
console.log("calling callback " + i)
callback();
console.log("here");
});
}, function(err) { //This is the final callback
if (err) return callback(err, null);
console.log("entering final callback");
// filter list
var resultList = [];
for(var i=0;i<newList.length;i++){
var priceRg = /(?:-|\.|,|\d)*$/ig;
var res = priceRg.exec(newList[i].profit);
var profit = parseFloat(res[0]);
if(profitFrom != "" && profit < profitFrom){
continue;
}
if(profitTo != "" && profit > profitTo){
continue;
}
resultList.push(newList[i]);
}
console.log("calling final callback")
callback(err, resultList);
});
}
function getAmazonURL(country) {
var url = '';
switch (country) {
case 'at':
url = 'http://www.amazon.at/';
break;
case 'ca':
url = 'http://www.amazon.ca/';
break;
case 'fr':
url = 'http://www.amazon.fr/';
break;
case 'it':
url = 'http://www.amazon.it/';
break;
case 'es':
url = 'http://www.amazon.es/';
break;
case 'de':
url = 'http://www.amazon.de/';
break;
case 'us':
url = 'http://www.amazon.com/';
break;
default:
url = 'http://www.amazon.co.uk/';
break;
}
return url;
}
