Skip to content

Commit

Permalink
Actor scraper and Site ID bug fix
Browse files Browse the repository at this point in the history
  • Loading branch information
pops64 committed Jan 6, 2025
1 parent b635961 commit ee6530f
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 7 deletions.
8 changes: 6 additions & 2 deletions pkg/config/scraper_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,12 @@ func CheckMatchingSiteID(findSite ScraperConfig, searchList []ScraperConfig) boo
func SetSiteId(configList *[]ScraperConfig, customId string) {
for idx, siteconfig := range *configList {
if siteconfig.FileID == "" || customId != "" {
id := strings.TrimRight(siteconfig.URL, "/")
siteconfig.ID = strings.ToLower(id[strings.LastIndex(id, "/")+1:])
temp := strings.TrimRight(siteconfig.URL, "/")
id := temp[strings.LastIndex(temp, "/")+1:]
if customId == "realvr" {
id = id[:strings.Index(id, "-")-1]
}
siteconfig.ID = strings.ToLower(id)
} else {
siteconfig.ID = strings.ToLower(siteconfig.FileID)
}
Expand Down
12 changes: 7 additions & 5 deletions pkg/scrape/badoink.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,14 @@ func BadoinkSite(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out
// Scene ID - get from URL
tmp := strings.Split(sc.HomepageURL, "-")
sc.SiteID = strings.Replace(tmp[len(tmp)-1], "/", "", -1)
if ogSite {
sc.SceneID = scraperID + "-" + sc.SiteID
} else {
sc.SceneID = "realvr" + "-" + sc.SiteID

idPrefix := scraperID
if !ogSite {
idPrefix = "realvr"
}

sc.SceneID = idPrefix + "-" + sc.SiteID

// Title
e.ForEach(`h1.video-title`, func(id int, e *colly.HTMLElement) {
sc.Title = strings.TrimSpace(e.Text)
Expand Down Expand Up @@ -106,7 +108,7 @@ func BadoinkSite(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out
sc.ActorDetails = make(map[string]models.ActorDetails)
e.ForEach(`a.video-actor-link`, func(id int, e *colly.HTMLElement) {
sc.Cast = append(sc.Cast, strings.TrimSpace(e.Text))
sc.ActorDetails[strings.TrimSpace(e.Text)] = models.ActorDetails{Source: sc.ScraperID + " scrape", ProfileUrl: e.Request.AbsoluteURL(e.Attr("href"))}
sc.ActorDetails[strings.TrimSpace(e.Text)] = models.ActorDetails{Source: idPrefix + " scrape", ProfileUrl: e.Request.AbsoluteURL(e.Attr("href"))}
})

// Date
Expand Down

0 comments on commit ee6530f

Please sign in to comment.