UrlFetchApp returns different response from what I see in browser
I'm trying to scrape hotel prices from Booking.com, and now getting a weird response body that is different from what I am seeing on my browser in private mode.
When you run the Google Script below then you'll get '2178' JPY as a result, while just hitting the same url on the address bar in private mode, you'll see '2420' JPY.
function myfunction() {
var destid = '2658464';
var year = '2018';
var checkin_month = '12';
var checkin_monthday = '20';
var checkout_month = '12';
var checkout_monthday = '21';
var url = 'https://www.booking.com/searchresults.en-gb.html?checkin_year=' + year + '&checkin_month='
+ checkin_month + '&checkin_monthday=' + checkin_monthday + '&checkout_year=' + year + '&checkout_month=' + checkout_month + '&checkout_monthday=' + checkout_monthday
+ '&no_rooms=１&group_adults=1&group_children=0&dest_id=' + destid + '&dest_type=hotel&selected_currency=JPY';
Logger.log(url);
var html = UrlFetchApp.fetch(url).getContentText();
// Retrieve 'div' that contains the top search result
var res = Parser.data(html).from("<span class="sr-hotel__name").to("class="sr_item sr_item_new sr_item_default").build().trim();
var price;
if ( res.indexOf('sold out on') != -1 ) { //when sold out on specified date
Logger.log( 'SOLD OUT' );
} else if ( res.indexOf('Dormitory') != -1 && res.indexOf( destid ) != -1 ){ //if its dormitory room and contains specified destId
// retrieve price value in <b> tag
price = Parser.data(res).from("<b>n¥").to("</b>").build().trim().replace(new RegExp('[^0-9]', 'g'), '');
Logger.log('Price: ' + price);
} else {
Logger.log( 'SOLD OUT' );
}
}Basically, Booking.com shows discounted price according to loyalty level (called Genius Booker) of the user currently logging in. My script somehow retrieves discounted price, even though giving any credentials.
In most cases this my scripts works fine enough to show the original price, but for some hotels, it returns discounted price for Genius bookers.
How come do you think does this happen, and is there any way to force scripts always to return the original price?
Thanks for your time.
google-apps-script web-scraping booking.com-api
asked Nov 21 '18 at 13:16
pommepomme
40128
add a comment |
I'm trying to scrape hotel prices from Booking.com, and now getting a weird response body that is different from what I am seeing on my browser in private mode.
When you run the Google Script below then you'll get '2178' JPY as a result, while just hitting the same url on the address bar in private mode, you'll see '2420' JPY.
function myfunction() {
var destid = '2658464';
var year = '2018';
var checkin_month = '12';
var checkin_monthday = '20';
var checkout_month = '12';
var checkout_monthday = '21';
var url = 'https://www.booking.com/searchresults.en-gb.html?checkin_year=' + year + '&checkin_month='
+ checkin_month + '&checkin_monthday=' + checkin_monthday + '&checkout_year=' + year + '&checkout_month=' + checkout_month + '&checkout_monthday=' + checkout_monthday
+ '&no_rooms=１&group_adults=1&group_children=0&dest_id=' + destid + '&dest_type=hotel&selected_currency=JPY';
Logger.log(url);
var html = UrlFetchApp.fetch(url).getContentText();
// Retrieve 'div' that contains the top search result
var res = Parser.data(html).from("<span class="sr-hotel__name").to("class="sr_item sr_item_new sr_item_default").build().trim();
var price;
if ( res.indexOf('sold out on') != -1 ) { //when sold out on specified date
Logger.log( 'SOLD OUT' );
} else if ( res.indexOf('Dormitory') != -1 && res.indexOf( destid ) != -1 ){ //if its dormitory room and contains specified destId
// retrieve price value in <b> tag
price = Parser.data(res).from("<b>n¥").to("</b>").build().trim().replace(new RegExp('[^0-9]', 'g'), '');
Logger.log('Price: ' + price);
} else {
Logger.log( 'SOLD OUT' );
}
}Basically, Booking.com shows discounted price according to loyalty level (called Genius Booker) of the user currently logging in. My script somehow retrieves discounted price, even though giving any credentials.
In most cases this my scripts works fine enough to show the original price, but for some hotels, it returns discounted price for Genius bookers.
How come do you think does this happen, and is there any way to force scripts always to return the original price?
Thanks for your time.
google-apps-script web-scraping booking.com-api
asked Nov 21 '18 at 13:16
pommepomme
40128
1
What does your code look like that doesn't work? Please provide a Minimal, Complete, and Verifiable example that demonstrates the issue.
– Jacque
Nov 22 '18 at 7:45
Hi @Jacque Thank you for your attention! I just modified my whole question to provide code that reproduce the issue. Would love you to take a look and appreciate any help or guess.
– pomme
Nov 23 '18 at 12:35
add a comment |
I'm trying to scrape hotel prices from Booking.com, and now getting a weird response body that is different from what I am seeing on my browser in private mode.
When you run the Google Script below then you'll get '2178' JPY as a result, while just hitting the same url on the address bar in private mode, you'll see '2420' JPY.
function myfunction() {
var destid = '2658464';
var year = '2018';
var checkin_month = '12';
var checkin_monthday = '20';
var checkout_month = '12';
var checkout_monthday = '21';
var url = 'https://www.booking.com/searchresults.en-gb.html?checkin_year=' + year + '&checkin_month='
+ checkin_month + '&checkin_monthday=' + checkin_monthday + '&checkout_year=' + year + '&checkout_month=' + checkout_month + '&checkout_monthday=' + checkout_monthday
+ '&no_rooms=１&group_adults=1&group_children=0&dest_id=' + destid + '&dest_type=hotel&selected_currency=JPY';
Logger.log(url);
var html = UrlFetchApp.fetch(url).getContentText();
// Retrieve 'div' that contains the top search result
var res = Parser.data(html).from("<span class="sr-hotel__name").to("class="sr_item sr_item_new sr_item_default").build().trim();
var price;
if ( res.indexOf('sold out on') != -1 ) { //when sold out on specified date
Logger.log( 'SOLD OUT' );
} else if ( res.indexOf('Dormitory') != -1 && res.indexOf( destid ) != -1 ){ //if its dormitory room and contains specified destId
// retrieve price value in <b> tag
price = Parser.data(res).from("<b>n¥").to("</b>").build().trim().replace(new RegExp('[^0-9]', 'g'), '');
Logger.log('Price: ' + price);
} else {
Logger.log( 'SOLD OUT' );
}
}Basically, Booking.com shows discounted price according to loyalty level (called Genius Booker) of the user currently logging in. My script somehow retrieves discounted price, even though giving any credentials.
In most cases this my scripts works fine enough to show the original price, but for some hotels, it returns discounted price for Genius bookers.
How come do you think does this happen, and is there any way to force scripts always to return the original price?
Thanks for your time.
google-apps-script web-scraping booking.com-api
asked Nov 21 '18 at 13:16
pommepomme
40128
I'm trying to scrape hotel prices from Booking.com, and now getting a weird response body that is different from what I am seeing on my browser in private mode.
When you run the Google Script below then you'll get '2178' JPY as a result, while just hitting the same url on the address bar in private mode, you'll see '2420' JPY.
function myfunction() {
var destid = '2658464';
var year = '2018';
var checkin_month = '12';
var checkin_monthday = '20';
var checkout_month = '12';
var checkout_monthday = '21';
var url = 'https://www.booking.com/searchresults.en-gb.html?checkin_year=' + year + '&checkin_month='
+ checkin_month + '&checkin_monthday=' + checkin_monthday + '&checkout_year=' + year + '&checkout_month=' + checkout_month + '&checkout_monthday=' + checkout_monthday
+ '&no_rooms=１&group_adults=1&group_children=0&dest_id=' + destid + '&dest_type=hotel&selected_currency=JPY';
Logger.log(url);
var html = UrlFetchApp.fetch(url).getContentText();
// Retrieve 'div' that contains the top search result
var res = Parser.data(html).from("<span class="sr-hotel__name").to("class="sr_item sr_item_new sr_item_default").build().trim();
var price;
if ( res.indexOf('sold out on') != -1 ) { //when sold out on specified date
Logger.log( 'SOLD OUT' );
} else if ( res.indexOf('Dormitory') != -1 && res.indexOf( destid ) != -1 ){ //if its dormitory room and contains specified destId
// retrieve price value in <b> tag
price = Parser.data(res).from("<b>n¥").to("</b>").build().trim().replace(new RegExp('[^0-9]', 'g'), '');
Logger.log('Price: ' + price);
} else {
Logger.log( 'SOLD OUT' );
}
}Basically, Booking.com shows discounted price according to loyalty level (called Genius Booker) of the user currently logging in. My script somehow retrieves discounted price, even though giving any credentials.
In most cases this my scripts works fine enough to show the original price, but for some hotels, it returns discounted price for Genius bookers.
How come do you think does this happen, and is there any way to force scripts always to return the original price?
Thanks for your time.
function myfunction() {
var destid = '2658464';
var year = '2018';
var checkin_month = '12';
var checkin_monthday = '20';
var checkout_month = '12';
var checkout_monthday = '21';
var url = 'https://www.booking.com/searchresults.en-gb.html?checkin_year=' + year + '&checkin_month='
+ checkin_month + '&checkin_monthday=' + checkin_monthday + '&checkout_year=' + year + '&checkout_month=' + checkout_month + '&checkout_monthday=' + checkout_monthday
+ '&no_rooms=１&group_adults=1&group_children=0&dest_id=' + destid + '&dest_type=hotel&selected_currency=JPY';
Logger.log(url);
var html = UrlFetchApp.fetch(url).getContentText();
// Retrieve 'div' that contains the top search result
var res = Parser.data(html).from("<span class="sr-hotel__name").to("class="sr_item sr_item_new sr_item_default").build().trim();
var price;
if ( res.indexOf('sold out on') != -1 ) { //when sold out on specified date
Logger.log( 'SOLD OUT' );
} else if ( res.indexOf('Dormitory') != -1 && res.indexOf( destid ) != -1 ){ //if its dormitory room and contains specified destId
// retrieve price value in <b> tag
price = Parser.data(res).from("<b>n¥").to("</b>").build().trim().replace(new RegExp('[^0-9]', 'g'), '');
Logger.log('Price: ' + price);
} else {
Logger.log( 'SOLD OUT' );
}
}function myfunction() {
var destid = '2658464';
var year = '2018';
var checkin_month = '12';
var checkin_monthday = '20';
var checkout_month = '12';
var checkout_monthday = '21';
var url = 'https://www.booking.com/searchresults.en-gb.html?checkin_year=' + year + '&checkin_month='
+ checkin_month + '&checkin_monthday=' + checkin_monthday + '&checkout_year=' + year + '&checkout_month=' + checkout_month + '&checkout_monthday=' + checkout_monthday
+ '&no_rooms=１&group_adults=1&group_children=0&dest_id=' + destid + '&dest_type=hotel&selected_currency=JPY';
Logger.log(url);
var html = UrlFetchApp.fetch(url).getContentText();
// Retrieve 'div' that contains the top search result
var res = Parser.data(html).from("<span class="sr-hotel__name").to("class="sr_item sr_item_new sr_item_default").build().trim();
var price;
if ( res.indexOf('sold out on') != -1 ) { //when sold out on specified date
Logger.log( 'SOLD OUT' );
} else if ( res.indexOf('Dormitory') != -1 && res.indexOf( destid ) != -1 ){ //if its dormitory room and contains specified destId
// retrieve price value in <b> tag
price = Parser.data(res).from("<b>n¥").to("</b>").build().trim().replace(new RegExp('[^0-9]', 'g'), '');
Logger.log('Price: ' + price);
} else {
Logger.log( 'SOLD OUT' );
}
}google-apps-script web-scraping booking.com-api
google-apps-script web-scraping booking.com-api
asked Nov 21 '18 at 13:16
pommepomme
40128
asked Nov 21 '18 at 13:16
pommepomme
40128
edited Nov 23 '18 at 12:33
pomme
asked Nov 21 '18 at 13:16
pommepomme
40128
asked Nov 21 '18 at 13:16
pommepomme
40128
asked Nov 21 '18 at 13:16
pommepomme
40128
40128
1
What does your code look like that doesn't work? Please provide a Minimal, Complete, and Verifiable example that demonstrates the issue.
– Jacque
Nov 22 '18 at 7:45
Hi @Jacque Thank you for your attention! I just modified my whole question to provide code that reproduce the issue. Would love you to take a look and appreciate any help or guess.
– pomme
Nov 23 '18 at 12:35
add a comment |
1
What does your code look like that doesn't work? Please provide a Minimal, Complete, and Verifiable example that demonstrates the issue.
– Jacque
Nov 22 '18 at 7:45
Hi @Jacque Thank you for your attention! I just modified my whole question to provide code that reproduce the issue. Would love you to take a look and appreciate any help or guess.
– pomme
Nov 23 '18 at 12:35
1
1
What does your code look like that doesn't work? Please provide a Minimal, Complete, and Verifiable example that demonstrates the issue.
– Jacque
Nov 22 '18 at 7:45
What does your code look like that doesn't work? Please provide a Minimal, Complete, and Verifiable example that demonstrates the issue.
– Jacque
Nov 22 '18 at 7:45
Hi @Jacque Thank you for your attention! I just modified my whole question to provide code that reproduce the issue. Would love you to take a look and appreciate any help or guess.
– pomme
Nov 23 '18 at 12:35
Hi @Jacque Thank you for your attention! I just modified my whole question to provide code that reproduce the issue. Would love you to take a look and appreciate any help or guess.
– pomme
Nov 23 '18 at 12:35
add a comment |
0
active
oldest
votes
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53412914%2furlfetchapp-returns-different-response-from-what-i-see-in-browser%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
0
active
oldest
votes
0
active
oldest
votes
active
oldest
votes
active
oldest
votes
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53412914%2furlfetchapp-returns-different-response-from-what-i-see-in-browser%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
1
What does your code look like that doesn't work? Please provide a Minimal, Complete, and Verifiable example that demonstrates the issue.
– Jacque
Nov 22 '18 at 7:45
Hi @Jacque Thank you for your attention! I just modified my whole question to provide code that reproduce the issue. Would love you to take a look and appreciate any help or guess.
– pomme
Nov 23 '18 at 12:35