Python crawler sets cookies to solve the problem of websites intercepting and crawling ant short term rentals

# -*- coding: utf-8 -*-
import urllib
import re
from bs4 import BeautifulSoup
import codecs
 
url = '/guiyang/?map=no'
response=(url)
contents = ()
soup = BeautifulSoup(contents, "")
print 
print soup
#Short-Term Rental Names
for tag in soup.find_all('dd'):
 for name in tag.find_all(attrs={"class":"room-detail clearfloat"}):
 fname = ('p').get_text()
 print u'[Name of short-term rental]'', ('\n','').strip()

# -*- coding: utf-8 -*-
import urllib2
import re
from bs4 import BeautifulSoup
 
#crawler functions
def gydzf(url):
 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
 headers={"User-Agent":user_agent}
 request=(url,headers=headers)
 response=(request)
 contents = ()
 soup = BeautifulSoup(contents, "")
 for tag in soup.find_all('dd'):
 #Short-Term Rental Names
 for name in tag.find_all(attrs={"class":"room-detail clearfloat"}):
 fname = ('p').get_text()
 print u'[Name of short-term rental]'', ('\n','').strip()
 #Short-Term Rental Rates
 for price in tag.find_all(attrs={"class":"moy-b"}):
 string = ('p').get_text()
 fprice = ("[￥]+".decode("utf8"), "".decode("utf8"),string)
 fprice = fprice[0:5]
 print u'[Short-term rental prices]'', ('\n','').strip()
 # of ratings and comments
 for score in ('ul'):
 fscore = ('ul').get_text()
 print u'[Short term rental ratings/reviews/number of occupants]'', ('\n','').strip()
 #webpage link url
 url_dzf = (attrs={"target":"_blank"})
 urls = url_dzf.attrs['href']
 print u'[Web Link]', ('\n','').strip()
 urlss = '' + urls + ''
 print urlss
 
#main function
if __name__ == '__main__':
 i = 1
 while i<10:
 print u'Page number', i
 url = '/guiyang/' + str(i) + '/?map=no'
 gydzf(url)
 i = i+1
 else:
 print u"The end."

user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) ... Chrome/61.0.3163.100 Safari/537.36"
 cookie="mediav=%7B%22eid%22%3A%22387123...b3574ef2-21b9-11e8-b39c-1bc4029c43b8"
 headers={"User-Agent":user_agent,"Cookie":cookie}
 request=(url,headers=headers)
 response=(request)
 contents = ()
 soup = BeautifulSoup(contents, "")
 for tag1 in soup.find_all(attrs={"class":"main"}):

import urllib2
import re
from bs4 import BeautifulSoup
import codecs
import csv
 
c = open("","wb") #write Write
(codecs.BOM_UTF8)
writer = (c)
(["Name of short-term rental","Address.","Price.","Scoring.","Number of occupants","Price per capita"])
 
#Crawl for details
def getInfo(url,fname,fprice,fscore,users):
 # Through the browser developer mode to view access to the use of user_agent and cookie settings access to the headers (headers) to avoid anti-crawler, and every once in a while to run according to the developer in the cookie to change the code in the cookie
 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"
 cookie="mediav=%7B%22eid%22%3A%22387123%22eb7; mayi_uuid=1582009990674274976491; sid=42200298656434922.85.130.130"
 headers={"User-Agent":user_agent,"Cookie":cookie}
 request=(url,headers=headers)
 response=(request)
 contents = ()
 soup = BeautifulSoup(contents, "")
 #Short-Term Rentals Address
 for tag1 in soup.find_all(attrs={"class":"main"}):
 print u'Short term rental address:'
 for tag2 in tag1.find_all(attrs={"class":"desWord"}):
 address = ('p').get_text()
 print address
 # of occupants
 print u'Number of occupants:'
 for tag4 in tag1.find_all(attrs={"class":"w258"}):
 yy = ('span').get_text()
 print yy
 fname = ("utf-8")
 address = ("utf-8")
 fprice = ("utf-8")
 fscore = ("utf-8")
 fpeople = yy[2:3].encode("utf-8")
 ones = int(float(fprice))/int(float(fpeople))
 #Store locally
 ([fname,address,fprice,fscore,fpeople,ones])
 
#crawler functions
def gydzf(url):
 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
 headers={"User-Agent":user_agent}
 request=(url,headers=headers)
 response=(request)
 contents = ()
 soup = BeautifulSoup(contents, "")
 for tag in soup.find_all('dd'):
 #Short-Term Rental Names
 for name in tag.find_all(attrs={"class":"room-detail clearfloat"}):
 fname = ('p').get_text()
 print u'[Name of short-term rental]'', ('\n','').strip()
 #Short-Term Rental Rates
 for price in tag.find_all(attrs={"class":"moy-b"}):
 string = ('p').get_text()
 fprice = ("[￥]+".decode("utf8"), "".decode("utf8"),string)
 fprice = fprice[0:5]
 print u'[Short-term rental prices]'', ('\n','').strip()
 # of ratings and comments
 for score in ('ul'):
 fscore = ('ul').get_text()
 print u'[Short term rental ratings/reviews/number of occupants]'', ('\n','').strip()
 #webpage link url
 url_dzf = (attrs={"target":"_blank"})
 urls = url_dzf.attrs['href']
 print u'[Web Link]', ('\n','').strip()
 urlss = '' + urls + ''
 print urlss
 getInfo(urlss,fname,fprice,fscore,user_agent)
 
#main function
if __name__ == '__main__':
 i = 0
 while i<33:
 print u'Page number', (i+1)
 if(i==0):
 url = '/guiyang/?map=no'
 if(i>0):
 num = i+2 # Except that the first page is empty and the second page starts incrementing in order of 2
 url = '/guiyang/' + str(num) + '/?map=no'
 gydzf(url)
 i=i+1
 
()