时间戳

首页 > 解决方案 > Scrape Website 与 POSTMan 一起使用 C# 失败

c# - Scrape Website 与 POSTMan 一起使用 C# 失败

问题描述

嗨，我正在开发一个抓取网络应用程序，我需要在 [ this ] 1 个站点中抓取多个页面，以https://www.caleprocure.ca.gov/pages/Events-BS3/event-search.aspx开头 我创建了一个邮递员collection，如果我发送请求两次（第一次请求失败，然后再次单击发送它有效）

如果我在 C# WebAPI 控制器上实现相同，它总是失败

private async void GetResultsAsync(int maxRetry)
        {
            while (maxRetry >= 0)
            {
                var content = new FormUrlEncodedContent(new[]
                {
                    new KeyValuePair<string, string>("IF-TargetVerb", "GET"),
                    new KeyValuePair<string, string>("IF-TargetContent",
                        "[{\"Lbl\":\"attachmentWrapper\",\"Src\":\"div.InFlightAttachment:first\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[{\"Lbl\":\"attachmentLink\",\"Src\":\".\",\"Data\":\"text:href\",\"Children\":[]}]},{\"Lbl\":\"popupMessageContent\",\"Src\":\"span.InFlightPopup\",\"Data\":\"html\",\"Children\":[]},{\"Lbl\":\"pageTitle\",\"Src\":\"div[id*=\'div$ICField85\'] > span\",\"Data\":\"text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"if_error_items\",\"Src\":\"span[id=\'RESP_INQA_WK_AUC_INQ_MSG1\']:eq(0)\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"error_items\",\"Src\":\"span[id=\'RESP_INQA_WK_AUC_INQ_MSG1\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"hideSavedSearches\",\"Src\":\"#AUC_PREF_WK_AUC_PREF_SRCH_NAME\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"savedSearchLabelLabel\",\"Src\":\"#AUC_PREF_WK_AUC_PREF_SRCH_NAME_LBL\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"savedSearches\",\"Src\":\"#AUC_PREF_WK_AUC_PREF_SRCH_NAME\",\"Data\":\"name id value\",\"Children\":[{\"Lbl\":\"ptype\",\"Src\":\"option\",\"Data\":\"value text selected disabled\",\"Children\":[]}]},{\"Lbl\":\"eventNameLabel\",\"Src\":\"label[id=\'RESP_INQA_WK_ZZ_AUC_NAME_LBL\']\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"eventNameGrouping\",\"Src\":\"#RESP_INQA_WK_ZZ_AUC_NAME\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"eventName\",\"Src\":\"#RESP_INQA_WK_ZZ_AUC_NAME\",\"Data\":\"id name value disabled readonly class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"eventIdLabel\",\"Src\":\"label[id=\'RESP_INQA_WK_AUC_ID_LBL\']\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"eventIdGrouping\",\"Src\":\"#RESP_INQA_WK_AUC_ID\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"eventId\",\"Src\":\"#RESP_INQA_WK_AUC_ID\",\"Data\":\"id name value disabled readonly class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"businessUnitLabel\",\"Src\":\"div[id*=\'div$ICField199\'] > span\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"item15\",\"Src\":\"#RESP_INQA_WK_BUSINESS_UNIT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"businessUnit\",\"Src\":\"#RESP_INQA_WK_BUSINESS_UNIT\",\"Data\":\"id name value disabled class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"businessUnitSelector\",\"Src\":\"a[id^=\'RESP_INQA_WK_BUSINESS_UNIT$prompt\']\",\"Data\":\"id name\",\"Children\":[]},{\"Lbl\":\"businessUnitSubtitle\",\"Src\":\"#BUS_UNIT_TBL_FS_DESCR\",\"Data\":\"text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"eventStatusLabel\",\"Src\":\"label[id=\'RESP_INQA_WK_ZZ_EVENT_STATUS_LBL\']\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"item20\",\"Src\":\"select[id=\'RESP_INQA_WK_ZZ_EVENT_STATUS\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"eventStatus\",\"Src\":\"select[id=\'RESP_INQA_WK_ZZ_EVENT_STATUS\']\",\"Data\":\"name id value\",\"Children\":[{\"Lbl\":\"ptype\",\"Src\":\"option\",\"Data\":\"value text selected disabled\",\"Children\":[]}]},{\"Lbl\":\"publishedYearLabel\",\"Src\":\"label[id=\'RESP_INQA_WK_YEAR_LBL\']\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"item23\",\"Src\":\"select[id=\'RESP_INQA_WK_YEAR\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"publishedYear\",\"Src\":\"select[id=\'RESP_INQA_WK_YEAR\']\",\"Data\":\"name id value\",\"Children\":[{\"Lbl\":\"ptype\",\"Src\":\"option\",\"Data\":\"value text selected disabled\",\"Children\":[]}]},{\"Lbl\":\"datesHider\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_START_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"startDateFromLabel\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_START_DT_LBL\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"startDateFromGrouping\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_START_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"startDateFromHider\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_START_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"startDateFrom\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_START_DT\",\"Data\":\"id name value disabled readonly class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"startDateToLabel\",\"Src\":\"#RESP_INQA_WK_AUC_TO_START_DT_LBL\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"startDateToGrouping\",\"Src\":\"#RESP_INQA_WK_AUC_TO_START_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"startDateToHider\",\"Src\":\"#RESP_INQA_WK_AUC_TO_START_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"startDateTo\",\"Src\":\"#RESP_INQA_WK_AUC_TO_START_DT\",\"Data\":\"id name value disabled readonly class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"endDateFromLabel\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_END_DT_LBL\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"endDateFromGrouping\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_END_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"endDateFromHider\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_END_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"endDateFrom\",\"Src\":\"#RESP_INQA_WK_AUC_FROM_END_DT\",\"Data\":\"id name value disabled readonly class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"endDateToLabel\",\"Src\":\"#RESP_INQA_WK_AUC_TO_END_DT_LBL\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"endDateToGrouping\",\"Src\":\"#RESP_INQA_WK_AUC_TO_END_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"endDateToHider\",\"Src\":\"#RESP_INQA_WK_AUC_TO_END_DT\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"endDateTo\",\"Src\":\"#RESP_INQA_WK_AUC_TO_END_DT\",\"Data\":\"id name value disabled readonly class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"searchItemDescLabel\",\"Src\":\"label[id=\'RESP_INQA_WK_DESCR254_MIXED_LBL\']\",\"Data\":\"text for\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"searchItemDescGrouping\",\"Src\":\"#RESP_INQA_WK_DESCR254_MIXED\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"searchItemDesc\",\"Src\":\"#RESP_INQA_WK_DESCR254_MIXED\",\"Data\":\"id name value disabled readonly class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"categoryTH1\",\"Src\":\"span[id=\'RESP_DUMMY2_VW$0#0\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"categoryTH4\",\"Src\":\"span[id=\'RESP_DUMMY2_VW$0#1\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"categoriesRow\",\"Src\":\"tr[id^=\'trRESP_DUMMY2_VW$0_row\']\",\"Data\":\"null\",\"Children\":[{\"Lbl\":\"categoriesArea\",\"Src\":\"input[id^=\'AUC_HDR_CAT_WRK_CATEGORY_CD\']\",\"Data\":\"id name value disabled class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"categoriesAreaLookup\",\"Src\":\"a[id^=\'AUC_HDR_CAT_WRK_CATEGORY_CD$prompt\']\",\"Data\":\"id name\",\"Children\":[]},{\"Lbl\":\"descr\",\"Src\":\"span[id^=\'AUC_HDR_CAT_WRK_DESCR60$\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"categoryDeleteButton\",\"Src\":\"a[id^=\'RESP_DUMMY2_VW$delete$\']\",\"Data\":\"id name disabled\",\"Children\":[]}]},{\"Lbl\":\"subCategoriesTH1\",\"Src\":\"span[id=\'RESP_DUMMY2_VW$0#0\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"item49\",\"Src\":\"input[type=hidden][id^=\'RESP_DUMMY2_VW$selmh$\']\",\"Data\":\"id name\",\"Children\":[]},{\"Lbl\":\"subCategoryTH5\",\"Src\":\"span[id=\'RESP_DUMMY2_VW$0#1\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"AddCategoryAreaButton\",\"Src\":\"[id^=\'RESP_DUMMY2_VW$new$\']:last\",\"Data\":\"id name disabled role tabindex\",\"Children\":[]},{\"Lbl\":\"ServiceTitle\",\"Src\":\"div[id*=\'ZZ_RESP_DMY1_VWGP$0\'] td[class=\'PSLEVEL1GRIDLABEL\']:first\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"serviceTH1\",\"Src\":\"span[id=\'ZZ_RESP_DMY1_VW$0#0\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"serviceTH4\",\"Src\":\"span[id=\'ZZ_RESP_DMY1_VW$0#1\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"serviceRow\",\"Src\":\"tr[id^=\'trZZ_RESP_DMY1_VW$0_row\']\",\"Data\":\"null\",\"Children\":[{\"Lbl\":\"serviceArea\",\"Src\":\"input[id^=\'ZZ_PO_DOC_WRK_ZZ_SRVC_AREA_ID$\']\",\"Data\":\"id name value disabled class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"serviceAreaLookup\",\"Src\":\"a[id^=\'ZZ_PO_DOC_WRK_ZZ_SRVC_AREA_ID$prompt$\']\",\"Data\":\"id name\",\"Children\":[]},{\"Lbl\":\"county\",\"Src\":\"span[id^=\'ZZ_SA_VW_COUNTY$\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"deleteButton\",\"Src\":\"a[id^=\'ZZ_RESP_DMY1_VW$delete$\']\",\"Data\":\"id name disabled\",\"Children\":[]}]},{\"Lbl\":\"subServiceTH5\",\"Src\":\"[id=\'ZZ_RESP_DMY1_VW$0#1\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"NewServiceButton\",\"Src\":\"[id^=\'ZZ_RESP_DMY1_VW$new$\']:last\",\"Data\":\"id name disabled role tabindex\",\"Children\":[]},{\"Lbl\":\"LicenseClassifactionTitle\",\"Src\":\"div[id*=\'ZZ_RESP_DMY2_VWGP$0\'] td[class=\'PSLEVEL1GRIDLABEL\']:first\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"licenseTH1\",\"Src\":\"span[id=\'ZZ_RESP_DMY2_VW$0#0\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"licenseTH4\",\"Src\":\"span[id=\'ZZ_RESP_DMY2_VW$0#1\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"licenseRow\",\"Src\":\"tr[id^=\'trZZ_RESP_DMY2_VW$0_row\']\",\"Data\":\"null\",\"Children\":[{\"Lbl\":\"licenseArea\",\"Src\":\"input[id^=\'ZZ_PO_DOC_WRK_ZZ_LICENSE_CLASS\']\",\"Data\":\"id name value disabled class:tmpclass maxlength\",\"Children\":[]},{\"Lbl\":\"licenseAreaLookup\",\"Src\":\"a[id^=\'ZZ_PO_DOC_WRK_ZZ_LICENSE_CLASS$prompt$\']\",\"Data\":\"id name\",\"Children\":[]},{\"Lbl\":\"county\",\"Src\":\"span[id^=\'ZZ_CLS_CD_VW_DESCR254\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"licenseDeleteButton\",\"Src\":\"a[id^=\'ZZ_RESP_DMY2_VW$delete$\']\",\"Data\":\"id name disabled\",\"Children\":[]}]},{\"Lbl\":\"subLicenseTH1\",\"Src\":\"span[id=\'ZZ_RESP_DMY2_VW$0#0\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"subLicenseTH5\",\"Src\":\"span[id=\'ZZ_RESP_DMY2_VW$0#1\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"AddLicenseButton\",\"Src\":\"[id^=\'ZZ_RESP_DMY2_VW$new\']:last\",\"Data\":\"id name disabled role tabindex\",\"Children\":[]},{\"Lbl\":\"item65\",\"Src\":\"#AUC_PREF_WK_AUC_PREF_CLEAR_PB\",\"Data\":\"id name\",\"Children\":[]},{\"Lbl\":\"SaveCriteria\",\"Src\":\"a[id=\'AUC_PREF_WK_AUC_PREF_ADD_PB\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[{\"Lbl\":\"SaveCriteriaButton\",\"Src\":\".\",\"Data\":\"id name disabled\",\"Children\":[]},{\"Lbl\":\"SaveCriteriaText\",\"Src\":\".\",\"Data\":\"text value:data-value\",\"Children\":[]}]},{\"Lbl\":\"AdvancedSearch\",\"Src\":\"a[id=\'RESP_INQA_WK_AUC_SRCH_ADVBSC_PB\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[{\"Lbl\":\"AdvancedSearchButton\",\"Src\":\".\",\"Data\":\"id name disabled\",\"Children\":[]},{\"Lbl\":\"AdvancedSearchText\",\"Src\":\".\",\"Data\":\"text value:data-value\",\"Children\":[]}]},{\"Lbl\":\"Search\",\"Src\":\"input[id=\'RESP_INQA_WK_INQ_AUC_GO_PB\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[{\"Lbl\":\"SearchButton\",\"Src\":\".\",\"Data\":\"id name disabled\",\"Children\":[]},{\"Lbl\":\"SearchText\",\"Src\":\".\",\"Data\":\"value:text\",\"Children\":[]}]},{\"Lbl\":\"eventtable\",\"Src\":\"table[id*=\'$scroll$0\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"pagergrPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW_GR$hmodal$0\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"firstPagegrPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW_GR$htop$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"prevPagegrPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW_GR$hup$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"pagePosgrPager\",\"Src\":\"div[id*=\'RESP_INQA_HD_VW_GRGP\'] span[class=PSGRIDCOUNTER]\",\"Data\":\"text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"viewAllgrPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW_GR$hviewall$0\']\",\"Data\":\"id name text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"downloadSearchgrPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW_GR$hexcel$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"nextPagegrPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW_GR$hdown$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"lastPagegrPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW_GR$hend$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"pagervwPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW$hmodal$0\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"firstPagevwPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW$htop$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"prevPagevwPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW$hup$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"pagePosvwPager\",\"Src\":\"div[id*=\'RESP_INQA_HD_VWGP\'] span[class=PSGRIDCOUNTER]\",\"Data\":\"text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"viewAllvwPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW$hviewall$0\']\",\"Data\":\"id name text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"downloadSearchvwPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW$hexcel$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"nextPagevwPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW$hdown$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"lastPagevwPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW$hend$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"tbl\",\"Src\":\"[id=\'l0RESP_INQA_HD_VW$0\'],[id=\'l0RESP_INQA_HD_VW_GR$0\']\",\"Data\":\"id\",\"Children\":[{\"Lbl\":\"tblHeadTr\",\"Src\":\"[id=\'l1RESP_INQA_HD_VW$0\'],[id=\'l1RESP_INQA_HD_VW_GR$0\']\",\"Data\":\"id\",\"Children\":[{\"Lbl\":\"thEventId\",\"Src\":\"[id=\'thRESP_INQA_HD_VW$0#2\'],[id=\'thRESP_INQA_HD_VW_GR$0#2\']\",\"Data\":\"id:data-key\",\"Children\":[{\"Lbl\":\"thSpan\",\"Src\":\"[id=\'RESP_INQA_HD_VW$0#2\'],[id=\'RESP_INQA_HD_VW_GR$0#2\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"thup\",\"Src\":\"img[src*=\'UP\']\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"thdn\",\"Src\":\"img[src*=\'DN\']\",\"HWA\":\".\",\"Children\":[]}]},{\"Lbl\":\"thEventName\",\"Src\":\"[id=\'thRESP_INQA_HD_VW$0#3\'],[id=\'thRESP_INQA_HD_VW_GR$0#3\']\",\"Data\":\"id:data-key\",\"Children\":[{\"Lbl\":\"thSpan\",\"Src\":\"[id=\'RESP_INQA_HD_VW$0#3\'],[id=\'RESP_INQA_HD_VW_GR$0#3\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"thup\",\"Src\":\"img[src*=\'UP\']\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"thdn\",\"Src\":\"img[src*=\'DN\']\",\"HWA\":\".\",\"Children\":[]}]},{\"Lbl\":\"thDepName\",\"Src\":\"[id=\'thRESP_INQA_HD_VW$0#1\'],[id=\'thRESP_INQA_HD_VW_GR$0#1\']\",\"Data\":\"id:data-key\",\"Children\":[{\"Lbl\":\"thSpan\",\"Src\":\"[id=\'RESP_INQA_HD_VW$0#1\'],[id=\'RESP_INQA_HD_VW_GR$0#1\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"thup\",\"Src\":\"img[src*=\'UP\']\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"thdn\",\"Src\":\"img[src*=\'DN\']\",\"HWA\":\".\",\"Children\":[]}]},{\"Lbl\":\"thPubDate\",\"Src\":\"[id=\'thRESP_INQA_HD_VW$0#6\'],[id=\'thRESP_INQA_HD_VW_GR$0#6\']\",\"Data\":\"id:data-key\",\"HWA\":\".\",\"Children\":[{\"Lbl\":\"thSpan\",\"Src\":\"[id=\'RESP_INQA_HD_VW$0#6\'],[id=\'RESP_INQA_HD_VW_GR$0#6\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"thup\",\"Src\":\"img[src*=\'UP\']\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"thdn\",\"Src\":\"img[src*=\'DN\']\",\"HWA\":\".\",\"Children\":[]}]},{\"Lbl\":\"thEndDate\",\"Src\":\"[id=\'thRESP_INQA_HD_VW$0#7\'],[id=\'thRESP_INQA_HD_VW_GR$0#7\']\",\"Data\":\"id:data-key\",\"Children\":[{\"Lbl\":\"thSpan\",\"Src\":\"[id=\'RESP_INQA_HD_VW$0#7\'],[id=\'RESP_INQA_HD_VW_GR$0#7\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"thup\",\"Src\":\"img[src*=\'UP\']\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"thdn\",\"Src\":\"img[src*=\'DN\']\",\"HWA\":\".\",\"Children\":[]}]},{\"Lbl\":\"thStatus\",\"Src\":\"[id=\'thRESP_INQA_HD_VW$0#8\'],[id=\'thRESP_INQA_HD_VW_GR$0#8\']\",\"Data\":\"id:data-key\",\"Children\":[{\"Lbl\":\"thSpan\",\"Src\":\"[id=\'RESP_INQA_HD_VW$0#8\'],[id=\'RESP_INQA_HD_VW_GR$0#8\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"thup\",\"Src\":\"img[src*=\'UP\']\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"thdn\",\"Src\":\"img[src*=\'DN\']\",\"HWA\":\".\",\"Children\":[]}]}]},{\"Lbl\":\"tblBodyTr\",\"Src\":\"tr[id^=\'trRESP_INQA_HD_VW_GR$0_row\'],tr[id^=\'trRESP_INQA_HD_VW$0_row\']\",\"Children\":[{\"Lbl\":\"tdEventId\",\"Src\":\"a[id^=\'AUC_ID_COL$\'],a[id^=\'AUC_ID_BUS_UNIT$\']\",\"Data\":\"text id name\",\"Children\":[]},{\"Lbl\":\"tdEventName\",\"Src\":\"[id^=\'RESP_INQA1_WK_ZZ_AUC_NAME$\'],[id^=\'RESP_INQA_HD_VW_ZZ_AUC_NAME$\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"tdDepName\",\"Src\":\"[id^=\'BUS_UNIT_TBL_FS_DESCR$\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"tdPubDate\",\"Src\":\"[id^=\'AUC_DTTM_FINISH_FR$\']\",\"Data\":\"text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"tdEndDate\",\"Src\":\"[id^=\'RESP_INQA1_WK_AUC_DTTM_FINISH$\'],[id^=\'RESP_INQA_HD_VW_AUC_DTTM_FINISH$\']\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"tdStatus\",\"Src\":\"[id^=\'ZZ_DERIVED_DESCR\']\",\"Data\":\"text\",\"Children\":[]}]}]},{\"Lbl\":\"pagergrPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW_GR$hmodal$0\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"firstPagegrPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW_GR$htop$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"prevPagegrPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW_GR$hup$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"pagePosgrPager\",\"Src\":\"div[id*=\'RESP_INQA_HD_VW_GRGP\'] span[class=PSGRIDCOUNTER]\",\"Data\":\"text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"viewAllgrPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW_GR$hviewall$0\']\",\"Data\":\"id name text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"downloadSearchgrPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW_GR$hexcel$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"nextPagegrPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW_GR$hdown$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"lastPagegrPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW_GR$hend$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"pagervwPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW$hmodal$0\']\",\"Data\":\"null\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"firstPagevwPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW$htop$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"prevPagevwPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW$hup$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"pagePosvwPager\",\"Src\":\"div[id*=\'RESP_INQA_HD_VWGP\'] span[class=PSGRIDCOUNTER]\",\"Data\":\"text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"viewAllvwPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW$hviewall$0\']\",\"Data\":\"id name text\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"downloadSearchvwPager\",\"Src\":\"[id=\'RESP_INQA_HD_VW$hexcel$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"nextPagevwPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW$hdown$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"lastPagevwPager\",\"Src\":\"a[id=\'RESP_INQA_HD_VW$hend$0\']\",\"Data\":\"id name\",\"HWA\":\".\",\"Children\":[]},{\"Lbl\":\"doPortalUrlText\",\"Src\":\"script:contains(\'DoPortalUrl\')\",\"Data\":\"text\",\"Children\":[]},{\"Lbl\":\"hiddenInput\",\"Src\":\"input[type=hidden]\",\"Data\":\"id name value\",\"Children\":[]}]"),
                    new KeyValuePair<string, string>("IF-Template", "/pages/Events-BS3/event-search.aspx"),
                    new KeyValuePair<string, string>("IF-IgnoreContent", string.Empty),
                });

                HttpClient client = new HttpClient();
                client.DefaultRequestHeaders.Add("Accept", "*/*");
                client.DefaultRequestHeaders.Add("User-Agent",
                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36");
                client.DefaultRequestHeaders.Add("Referer",
                    "https://www.caleprocure.ca.gov/pages/Events-BS3/event-search.aspx");
                client.DefaultRequestHeaders.Add("Origin", "https://www.caleprocure.ca.gov");
                client.DefaultRequestHeaders.Add("Host", "www.caleprocure.ca.gov");

                var mainResponse = await client.PostAsync(_baseUrl, content);
                var mainPageContents = await mainResponse.Content.ReadAsStringAsync();
                if (mainResponse.StatusCode != HttpStatusCode.OK)
                    continue;
                var jsonResponse = JObject.Parse(mainPageContents);
                var captureResults = (JObject) jsonResponse["CaptureResults"];
                JArray elements = new JArray();
                var tblBodyTr = (JArray) captureResults["tbl"]["Children"]["tblBodyTr"];
                foreach (var tr in tblBodyTr)
                {
                    var element = (JObject) tr["Children"]["tdEventId"][0]["Properties"];
                    elements.Add(element);
                }

                maxRetry--;
            }
        }

还有什么是模拟点击此页面中每个结果的最佳方法，从该详细信息页面抓取数据？

任何帮助将不胜感激。

当它失败时，它会返回 278 状态代码和以下 JSON 响应

{"IFLocation":"/--IF/PD1/InFlightViewer1/68/DisplayLang=ENG/NLX3/FI--/psc/psfpd1/SUPPLIER/ERP/c/AUC_MANAGE_BIDS.AUC_RESP_INQ_AUC.GBL?bCDC=y&"}

当它成功时，它返回 200 状态码和一个大的 JSON 响应，如下所示（在主节点上折叠以适合图片） 在此处输入图像描述

标签： c#web-scrapingpostman

解决方案

推荐阅读