Skip to content

Commit 3aa7263

Browse files
TriPSsTriPSs
committed
fix: Some overal fixes and log improvements
1 parent e651bd5 commit 3aa7263

File tree

14 files changed

+84
-68
lines changed

14 files changed

+84
-68
lines changed

‎apps/api/src/shared/subtitles/subtitles.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ export class SubtitlesService {
112112
score: subtitle.score
113113
})
114114
} catch (err) {
115-
this.logger.error(`[${download._id}]: Could not download subtitle "${language}" (${subtitle.url})`, err)
115+
this.logger.error(`[${download._id}]: Could not download subtitle "${language}" (${subtitle.url})`, err.stack)
116116
}
117117
})
118118
)

‎libs/scraper/helpers/base/src/base.helper.ts

+5-19
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { Logger } from '@nestjs/common'
22
import { InjectModel } from '@nestjs/mongoose'
33
import { BlacklistModel, Movie, Show, Runtime } from '@pct-org/mongo-models'
44
import { ScrapedItem, ScrapedTorrent, ScrapedShowTorrents } from '@pct-org/scraper/providers/base'
5+
import { holder } from '@pct-org/constants/default-image-sizes'
56

67
/**
78
* Base class for scraping content from various sources.
@@ -13,21 +14,6 @@ export abstract class BaseHelper {
1314

1415
protected abstract readonly logger: Logger
1516

16-
/**
17-
* The default image link.
18-
*/
19-
static Holder: string = null
20-
21-
/**
22-
* The default image sizes object.
23-
*/
24-
static DefaultImageSizes = {
25-
full: BaseHelper.Holder,
26-
high: BaseHelper.Holder,
27-
medium: BaseHelper.Holder,
28-
thumb: BaseHelper.Holder
29-
}
30-
3117
public abstract getItem(imdb?: string, slug?: string): Promise<Movie | Show | undefined>
3218

3319
public abstract shouldUpdateExistingItem(item: Movie | Show): boolean
@@ -83,11 +69,11 @@ export abstract class BaseHelper {
8369
let expires = 0
8470

8571
if (until) {
86-
this.logger.warn(`Adding "${content.title}" with identifier "${content.slug}" to the blacklist until '${until}' because of reason '${reason}'`)
72+
this.logger.debug(`Adding "${content.title}" with identifier "${content.slug}" to the blacklist until '${until}' because of reason '${reason}'`)
8773
expires = Number(until)
8874

8975
} else if (weeks) {
90-
this.logger.warn(`Adding "${content.title}" with identifier "${content.slug}" to the blacklist for ${weeks} weeks because of reason '${reason}'`)
76+
this.logger.debug(`Adding "${content.title}" with identifier "${content.slug}" to the blacklist for ${weeks} weeks because of reason '${reason}'`)
9177
expires = Number(new Date(Date.now() + (6.04e+8 * weeks)))
9278
}
9379

@@ -107,13 +93,13 @@ export abstract class BaseHelper {
10793
*/
10894
protected checkImages(item: Show | Movie): Promise<Show | Movie> {
10995
for (const i in item.images.backdrop) {
110-
if (item.images.backdrop[i] === BaseHelper.Holder) {
96+
if (item.images.backdrop[i] === holder) {
11197
return Promise.reject(item)
11298
}
11399
}
114100

115101
for (const i in item.images.poster) {
116-
if (item.images.poster[i] === BaseHelper.Holder) {
102+
if (item.images.poster[i] === holder) {
117103
return Promise.reject(item)
118104
}
119105
}

‎libs/scraper/helpers/movie/src/movie-helper.service.ts

+15-10
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { TmdbService } from '@pct-org/services/tmdb'
88
import { FanartService } from '@pct-org/services/fanart'
99
import { OmdbService } from '@pct-org/services/omdb'
1010
import { formatTorrents } from '@pct-org/torrent/utils'
11+
import { defaultMovieImages } from '@pct-org/constants/default-image-sizes'
1112

1213
@Injectable()
1314
export class MovieHelperService extends BaseHelper {
@@ -30,8 +31,18 @@ export class MovieHelperService extends BaseHelper {
3031
protected readonly logger = new Logger('MovieHelper')
3132

3233
public async getItem(imdb: string = null, slug: string = null): Promise<Movie | undefined> {
34+
const or = []
35+
36+
if (imdb) {
37+
or.push({ _id: imdb })
38+
}
39+
40+
if (slug) {
41+
or.push({ slug })
42+
}
43+
3344
return this.movieModel.findOne({
34-
[imdb ? '_id' : 'slug']: imdb || slug
45+
$or: or
3546
},
3647
{},
3748
{ lean: true }
@@ -78,20 +89,17 @@ export class MovieHelperService extends BaseHelper {
7889
const traktMovie = await this.traktService.getMovieSummary(item.imdb)
7990

8091
if (!traktMovie) {
81-
// Try again in 1 week
82-
await this.addToBlacklist(item, MovieType, '404', 1)
83-
8492
return
8593
}
8694

8795
const traktWatchers = await this.traktService.getMovieWatching(item.imdb)
8896
const ratingPercentage = Math.round(traktMovie.rating * 10)
8997

9098
return {
91-
_id: traktMovie.ids.imdb,
99+
_id: traktMovie.ids.imdb || item.imdb,
92100
imdbId: traktMovie.ids.imdb,
93101
tmdbId: traktMovie.ids.tmdb,
94-
slug: traktMovie.ids.slug,
102+
slug: item.slug,
95103
title: traktMovie.title,
96104
released: new Date(traktMovie.released).getTime(),
97105
certification: traktMovie.certification,
@@ -103,10 +111,7 @@ export class MovieHelperService extends BaseHelper {
103111
watching: traktWatchers?.length ?? 0,
104112
percentage: ratingPercentage
105113
},
106-
images: {
107-
backdrop: BaseHelper.DefaultImageSizes,
108-
poster: BaseHelper.DefaultImageSizes
109-
},
114+
images: defaultMovieImages,
110115
genres: traktMovie.genres
111116
? traktMovie.genres
112117
: ['unknown'],

‎libs/scraper/helpers/season/src/season-helper.service.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ export class SeasonHelperService {
3333
}
3434

3535
traktSeasons.forEach((traktSeason) => {
36-
const formattedEpisodes = traktSeason.episodes.map((traktEpisode) => (
36+
const formattedEpisodes = traktSeason?.episodes?.map((traktEpisode) => (
3737
this.episodeHelperService.formatTraktEpisode(
3838
show,
3939
traktEpisode,
4040
torrents?.[traktEpisode.season]?.[traktEpisode.number] ?? []
4141
)
42-
))
42+
)) ?? []
4343

4444
// Remove it from the episodes seasons
4545
seasonNumbers = seasonNumbers.filter((season) => parseInt(season, 10) !== traktSeason.number)

‎libs/scraper/helpers/show/src/show-helper.service.ts

+17-10
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { TmdbService } from '@pct-org/services/tmdb'
88
import { FanartService } from '@pct-org/services/fanart'
99
import { TvdbService } from '@pct-org/services/tvdb'
1010
import { SeasonHelperService } from '@pct-org/scraper/helpers/season'
11+
import { defaultShowImages } from '@pct-org/constants/default-image-sizes'
1112

1213
@Injectable()
1314
export class ShowHelperService extends BaseHelper {
@@ -33,16 +34,26 @@ export class ShowHelperService extends BaseHelper {
3334
protected readonly logger = new Logger('ShowHelper')
3435

3536
public async getItem(imdb?: string, slug?: string): Promise<Show | undefined> {
37+
const or = []
38+
39+
if (imdb) {
40+
or.push({ _id: imdb })
41+
}
42+
43+
if (slug) {
44+
or.push({ slug })
45+
}
46+
3647
return this.showModel.findOne({
37-
[imdb ? '_id' : 'slug']: imdb || slug
48+
$or: or
3849
},
3950
{},
4051
{ lean: true }
4152
)
4253
}
4354

4455
public shouldUpdateExistingItem(item: Show): boolean {
45-
// Shows depends more heavily on the blacklist so we
56+
// Shows depends more heavily on the blacklist so
4657
// if we have a existing item we need to update it.
4758
return true
4859
}
@@ -86,8 +97,6 @@ export class ShowHelperService extends BaseHelper {
8697
}
8798

8899
if (!traktShow) {
89-
// Try again in 1 week
90-
await this.addToBlacklist(item, ShowType, '404', 1)
91100
return
92101
}
93102

@@ -114,8 +123,8 @@ export class ShowHelperService extends BaseHelper {
114123
await this.addShowToBlackListIfNeeded(item, traktShow, traktNextEpisode, traktLastEpisode)
115124

116125
return {
117-
_id: traktShow.ids.imdb,
118-
slug: traktShow.ids.slug,
126+
_id: traktShow.ids.imdb || item.imdb,
127+
slug: item.slug,
119128
imdbId: traktShow.ids.imdb,
120129
tmdbId: traktShow.ids.tmdb,
121130
tvdbId: traktShow.ids.tvdb,
@@ -132,10 +141,7 @@ export class ShowHelperService extends BaseHelper {
132141
watching: traktWatchers?.length ?? 0,
133142
percentage: ratingPercentage
134143
},
135-
images: {
136-
backdrop: BaseHelper.DefaultImageSizes,
137-
poster: BaseHelper.DefaultImageSizes
138-
},
144+
images: defaultShowImages,
139145
type: ShowType,
140146
genres: traktShow.genres
141147
? traktShow.genres
@@ -258,6 +264,7 @@ export class ShowHelperService extends BaseHelper {
258264
* - The show has a airing episode in the future
259265
*/
260266
private async addShowToBlackListIfNeeded(item: Pick<ScrapedItem, 'imdb' | 'slug' | 'title'>, show: TraktShow, nextEpisode?: TraktEpisode, lastEpisode?: TraktEpisode) {
267+
// Show is ended or canceled
261268
if (['ended', 'canceled'].includes(show.status)) {
262269
await this.addToBlacklist(
263270
item,

‎libs/scraper/providers/base/src/base.provider.ts

+16-5
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import {
1717
ScraperProviderConfigRegex
1818
} from './base.interfaces'
1919

20-
2120
/**
2221
* Base class for scraping content from various sources.
2322
*/
@@ -125,7 +124,7 @@ export abstract class BaseProvider {
125124
} catch (err) {
126125
const errorMessage = err.message || err
127126

128-
this.logger.error(`BaseProvider.scrapeConfig: ${errorMessage}`, err.trace)
127+
this.logger.error(`BaseProvider.scrapeConfig: ${errorMessage}`, err.stack)
129128

130129
// Log the content so it can be better debugged from logs
131130
if (errorMessage.includes('Could not find any data with slug')) {
@@ -139,7 +138,7 @@ export abstract class BaseProvider {
139138
}
140139
)
141140
} catch (err) {
142-
this.logger.error(`Catch BaseProvider.scrapeConfig: ${err.message || err}`, err.trace)
141+
this.logger.error(`Catch BaseProvider.scrapeConfig: ${err.message || err}`, err.stack)
143142
}
144143
}
145144

@@ -151,13 +150,18 @@ export abstract class BaseProvider {
151150
protected async isItemBlackListed(content: ScrapedItem): Promise<boolean | Error> {
152151
const { slug, imdb } = content
153152

154-
const blacklistedItem = await this.blackListModel.findOne({ _id: imdb || slug })
153+
const blacklistedItem = await this.blackListModel.findOne({
154+
$or: [
155+
{ _id: imdb },
156+
{ _id: slug }
157+
]
158+
})
155159

156160
if (blacklistedItem) {
157161
if (blacklistedItem.get('expires') > Date.now()) {
158162
const expires = new Date(blacklistedItem.get('expires'))
159163

160-
this.logger.warn(
164+
this.logger.debug(
161165
`'${imdb || slug}' is in the blacklist until '${expires}' because of reason '${blacklistedItem.get('reason')}', skipping...`
162166
)
163167

@@ -189,6 +193,13 @@ export abstract class BaseProvider {
189193
return Promise.reject(new Error(`'${this.contentType}' is not a valid value for ContentTypes!`))
190194
}
191195

196+
// Double check if the item has torrents
197+
if (!item.torrents || item.torrents.length === 0) {
198+
this.logger.warn(`'${item.slug}' has no torrents`)
199+
200+
return Promise.resolve()
201+
}
202+
192203
const existingItem = await helper.getItem(item.imdb, item.slug)
193204
const updateExistingItem = existingItem && helper.shouldUpdateExistingItem(existingItem)
194205

‎libs/scraper/providers/eztv/src/eztv-provider.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ export class EztvProviderService extends BaseProvider {
5252
} catch (err) {
5353
const errorMessage = err.message || err
5454

55-
this.logger.error(`EztvProviderService.scrapeConfig: ${errorMessage}`, err.trace)
55+
this.logger.error(`EztvProviderService.scrapeConfig: ${errorMessage}`, err.stack)
5656

5757
// Log the content so it can be better debugged from logs
5858
if (errorMessage.includes('Could not find any data with slug')) {

‎libs/scraper/providers/yts/src/yts-provider.service.ts

-4
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,6 @@ export class YtsProviderService extends BaseProvider {
3636
this.api = yts
3737
}
3838

39-
protected async getTotalPages(): Promise<number> {
40-
return 10
41-
}
42-
4339
/**
4440
* Get content info from a given torrent
4541
*/

‎libs/services/fanart/src/fanart.service.ts

+12-2
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,12 @@ export class FanartService {
3737
: null
3838

3939
} catch (err) {
40-
this.logger.error(`Error happened getting images for '${item.slug}'`, err)
40+
if (err.message.includes('404')) {
41+
this.logger.warn(`Can't find images for slug '${item.slug}'`)
42+
43+
} else {
44+
this.logger.error(`Error happened getting images for '${item.slug}' "${err.message}"`, err.stack)
45+
}
4146
}
4247

4348
return {
@@ -69,7 +74,12 @@ export class FanartService {
6974
: null
7075

7176
} catch (err) {
72-
this.logger.error(`Error happened getting images for '${item.slug}'`, err)
77+
if (err.message.includes('404')) {
78+
this.logger.warn(`Can't find images for slug '${item.slug}'`)
79+
80+
} else {
81+
this.logger.error(`Error happened getting images for '${item.slug}' "${err.message}"`, err.stack)
82+
}
7383
}
7484

7585
return {

‎libs/services/omdb/src/omdb.service.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export class OmdbService {
2222
public async getMovieImages(item: Movie): Promise<Images> {
2323
let poster = null
2424

25+
// If we already have a poster we can skip this one
2526
if (item.images.poster.full) {
2627
return item.images
2728
}
@@ -36,7 +37,7 @@ export class OmdbService {
3637

3738
} catch (err) {
3839
if (err.statusCode && err.statusCode === 404) {
39-
this.logger.error(`Can't find images for '${item.slug}'`, err)
40+
this.logger.warn(`Can't find images for '${item.slug}'`)
4041

4142
} else if (err.statusCode && err.statusCode === 401) {
4243
this.logger.error(`Rate limit hit when fetching for '${item.slug}'`, err)

‎libs/services/tmdb/src/tmdb.service.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ export class TmdbService {
3333

3434
} catch (err) {
3535
if (err instanceof NotFoundError) {
36-
this.logger.warn(`Can't find season for slug '${item.slug}' season '${season.number}'`)
36+
this.logger.warn(`Can't find season for slug '${item.slug}' season '${season.number}'`)
3737

3838
} else {
39-
this.logger.error(`Error happened getting season info for '${item.slug}' and season '${season.number}'`, err)
39+
this.logger.error(`Error happened getting season info for '${item.slug}' and season '${season.number}'`, err)
4040
}
4141

4242
throw err

‎libs/services/trakt/src/interfaces/trakt.season.interface.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,6 @@ export interface TraktSeason {
3030

3131
network: string
3232

33-
episodes: TraktEpisode[]
33+
episodes?: TraktEpisode[]
3434

3535
}

0 commit comments

Comments
 (0)