From 2ba3be4c59197a84083e9d913a71ac5a9a740f63 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Sun, 2 Jun 2024 23:38:38 +0800 Subject: [PATCH] feat: add api for subjects browsing (#564) Co-authored-by: Trim21 --- Taskfile.yaml | 25 ++++ internal/cachekey/cachekey.go | 8 ++ internal/mocks/SubjectCachedRepo.go | 118 +++++++++++++++++ internal/mocks/SubjectRepo.go | 118 +++++++++++++++++ internal/pkg/gstr/parse.go | 24 ++++ internal/subject/cache_repo.go | 65 ++++++++++ internal/subject/domain.go | 49 +++++++ internal/subject/mysql_repository.go | 88 +++++++++++++ internal/subject/mysql_repository_test.go | 59 +++++++++ openapi/components/subject_cat_anime.yaml | 31 +++++ openapi/components/subject_cat_book.yaml | 27 ++++ openapi/components/subject_cat_game.yaml | 31 +++++ openapi/components/subject_cat_real.yaml | 43 +++++++ openapi/components/subject_v0_slim.yaml | 6 +- openapi/v0.yaml | 115 ++++++++++++++++- readme.md | 4 +- web/handler/subject/browse.go | 150 ++++++++++++++++++++++ web/handler/subject/subject.go | 1 + web/req/query_parse.go | 19 +++ 19 files changed, 975 insertions(+), 6 deletions(-) create mode 100644 openapi/components/subject_cat_anime.yaml create mode 100644 openapi/components/subject_cat_book.yaml create mode 100644 openapi/components/subject_cat_game.yaml create mode 100644 openapi/components/subject_cat_real.yaml create mode 100644 web/handler/subject/browse.go diff --git a/Taskfile.yaml b/Taskfile.yaml index 5f8d4ed0b..ff17b7a14 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -44,6 +44,31 @@ tasks: env: CGO_ENABLED: "0" + web: + desc: Run Web Server + aliases: + - serve + - server + cmds: + - go run main.go --config config.yaml web + + consumer: + desc: Run Kafka Consumer + aliases: + - canal + cmds: + - go run main.go canal --config config.yaml + + openapi-test: + desc: Test OpenAPI Schema + cmds: + - npm run test + + openapi: + desc: Build OpenAPI Schema + cmds: + - npm run build + bench: desc: Run benchmark cmds: diff --git a/internal/cachekey/cachekey.go b/internal/cachekey/cachekey.go index 99ce587f6..319cc3b23 100644 --- a/internal/cachekey/cachekey.go +++ b/internal/cachekey/cachekey.go @@ -35,6 +35,14 @@ func Subject(id model.SubjectID) string { return resPrefix + "subject:" + strconv.FormatUint(uint64(id), 10) } +func SubjectBrowse(s string, limit, offset int) string { + return resPrefix + "subject::browse:" + s + ":" + strconv.Itoa(limit) + ":" + strconv.Itoa(offset) +} + +func SubjectBrowseCount(s string) string { + return resPrefix + "subject::browse:" + s + "::count" +} + func Episode(id model.EpisodeID) string { return resPrefix + "episode:" + strconv.FormatUint(uint64(id), 10) } diff --git a/internal/mocks/SubjectCachedRepo.go b/internal/mocks/SubjectCachedRepo.go index 867524561..48c1ce036 100644 --- a/internal/mocks/SubjectCachedRepo.go +++ b/internal/mocks/SubjectCachedRepo.go @@ -26,6 +26,124 @@ func (_m *SubjectCachedRepo) EXPECT() *SubjectCachedRepo_Expecter { return &SubjectCachedRepo_Expecter{mock: &_m.Mock} } +// Browse provides a mock function with given fields: ctx, filter, limit, offset +func (_m *SubjectCachedRepo) Browse(ctx context.Context, filter subject.BrowseFilter, limit int, offset int) ([]model.Subject, error) { + ret := _m.Called(ctx, filter, limit, offset) + + if len(ret) == 0 { + panic("no return value specified for Browse") + } + + var r0 []model.Subject + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, subject.BrowseFilter, int, int) ([]model.Subject, error)); ok { + return rf(ctx, filter, limit, offset) + } + if rf, ok := ret.Get(0).(func(context.Context, subject.BrowseFilter, int, int) []model.Subject); ok { + r0 = rf(ctx, filter, limit, offset) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]model.Subject) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, subject.BrowseFilter, int, int) error); ok { + r1 = rf(ctx, filter, limit, offset) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubjectCachedRepo_Browse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Browse' +type SubjectCachedRepo_Browse_Call struct { + *mock.Call +} + +// Browse is a helper method to define mock.On call +// - ctx context.Context +// - filter subject.BrowseFilter +// - limit int +// - offset int +func (_e *SubjectCachedRepo_Expecter) Browse(ctx interface{}, filter interface{}, limit interface{}, offset interface{}) *SubjectCachedRepo_Browse_Call { + return &SubjectCachedRepo_Browse_Call{Call: _e.mock.On("Browse", ctx, filter, limit, offset)} +} + +func (_c *SubjectCachedRepo_Browse_Call) Run(run func(ctx context.Context, filter subject.BrowseFilter, limit int, offset int)) *SubjectCachedRepo_Browse_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(subject.BrowseFilter), args[2].(int), args[3].(int)) + }) + return _c +} + +func (_c *SubjectCachedRepo_Browse_Call) Return(_a0 []model.Subject, _a1 error) *SubjectCachedRepo_Browse_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SubjectCachedRepo_Browse_Call) RunAndReturn(run func(context.Context, subject.BrowseFilter, int, int) ([]model.Subject, error)) *SubjectCachedRepo_Browse_Call { + _c.Call.Return(run) + return _c +} + +// Count provides a mock function with given fields: ctx, filter +func (_m *SubjectCachedRepo) Count(ctx context.Context, filter subject.BrowseFilter) (int64, error) { + ret := _m.Called(ctx, filter) + + if len(ret) == 0 { + panic("no return value specified for Count") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, subject.BrowseFilter) (int64, error)); ok { + return rf(ctx, filter) + } + if rf, ok := ret.Get(0).(func(context.Context, subject.BrowseFilter) int64); ok { + r0 = rf(ctx, filter) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, subject.BrowseFilter) error); ok { + r1 = rf(ctx, filter) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubjectCachedRepo_Count_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Count' +type SubjectCachedRepo_Count_Call struct { + *mock.Call +} + +// Count is a helper method to define mock.On call +// - ctx context.Context +// - filter subject.BrowseFilter +func (_e *SubjectCachedRepo_Expecter) Count(ctx interface{}, filter interface{}) *SubjectCachedRepo_Count_Call { + return &SubjectCachedRepo_Count_Call{Call: _e.mock.On("Count", ctx, filter)} +} + +func (_c *SubjectCachedRepo_Count_Call) Run(run func(ctx context.Context, filter subject.BrowseFilter)) *SubjectCachedRepo_Count_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(subject.BrowseFilter)) + }) + return _c +} + +func (_c *SubjectCachedRepo_Count_Call) Return(_a0 int64, _a1 error) *SubjectCachedRepo_Count_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SubjectCachedRepo_Count_Call) RunAndReturn(run func(context.Context, subject.BrowseFilter) (int64, error)) *SubjectCachedRepo_Count_Call { + _c.Call.Return(run) + return _c +} + // Get provides a mock function with given fields: ctx, id, filter func (_m *SubjectCachedRepo) Get(ctx context.Context, id uint32, filter subject.Filter) (model.Subject, error) { ret := _m.Called(ctx, id, filter) diff --git a/internal/mocks/SubjectRepo.go b/internal/mocks/SubjectRepo.go index 84c36ee4b..8ec7b2338 100644 --- a/internal/mocks/SubjectRepo.go +++ b/internal/mocks/SubjectRepo.go @@ -26,6 +26,124 @@ func (_m *SubjectRepo) EXPECT() *SubjectRepo_Expecter { return &SubjectRepo_Expecter{mock: &_m.Mock} } +// Browse provides a mock function with given fields: ctx, filter, limit, offset +func (_m *SubjectRepo) Browse(ctx context.Context, filter subject.BrowseFilter, limit int, offset int) ([]model.Subject, error) { + ret := _m.Called(ctx, filter, limit, offset) + + if len(ret) == 0 { + panic("no return value specified for Browse") + } + + var r0 []model.Subject + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, subject.BrowseFilter, int, int) ([]model.Subject, error)); ok { + return rf(ctx, filter, limit, offset) + } + if rf, ok := ret.Get(0).(func(context.Context, subject.BrowseFilter, int, int) []model.Subject); ok { + r0 = rf(ctx, filter, limit, offset) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]model.Subject) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, subject.BrowseFilter, int, int) error); ok { + r1 = rf(ctx, filter, limit, offset) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubjectRepo_Browse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Browse' +type SubjectRepo_Browse_Call struct { + *mock.Call +} + +// Browse is a helper method to define mock.On call +// - ctx context.Context +// - filter subject.BrowseFilter +// - limit int +// - offset int +func (_e *SubjectRepo_Expecter) Browse(ctx interface{}, filter interface{}, limit interface{}, offset interface{}) *SubjectRepo_Browse_Call { + return &SubjectRepo_Browse_Call{Call: _e.mock.On("Browse", ctx, filter, limit, offset)} +} + +func (_c *SubjectRepo_Browse_Call) Run(run func(ctx context.Context, filter subject.BrowseFilter, limit int, offset int)) *SubjectRepo_Browse_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(subject.BrowseFilter), args[2].(int), args[3].(int)) + }) + return _c +} + +func (_c *SubjectRepo_Browse_Call) Return(_a0 []model.Subject, _a1 error) *SubjectRepo_Browse_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SubjectRepo_Browse_Call) RunAndReturn(run func(context.Context, subject.BrowseFilter, int, int) ([]model.Subject, error)) *SubjectRepo_Browse_Call { + _c.Call.Return(run) + return _c +} + +// Count provides a mock function with given fields: ctx, filter +func (_m *SubjectRepo) Count(ctx context.Context, filter subject.BrowseFilter) (int64, error) { + ret := _m.Called(ctx, filter) + + if len(ret) == 0 { + panic("no return value specified for Count") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, subject.BrowseFilter) (int64, error)); ok { + return rf(ctx, filter) + } + if rf, ok := ret.Get(0).(func(context.Context, subject.BrowseFilter) int64); ok { + r0 = rf(ctx, filter) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, subject.BrowseFilter) error); ok { + r1 = rf(ctx, filter) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SubjectRepo_Count_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Count' +type SubjectRepo_Count_Call struct { + *mock.Call +} + +// Count is a helper method to define mock.On call +// - ctx context.Context +// - filter subject.BrowseFilter +func (_e *SubjectRepo_Expecter) Count(ctx interface{}, filter interface{}) *SubjectRepo_Count_Call { + return &SubjectRepo_Count_Call{Call: _e.mock.On("Count", ctx, filter)} +} + +func (_c *SubjectRepo_Count_Call) Run(run func(ctx context.Context, filter subject.BrowseFilter)) *SubjectRepo_Count_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(subject.BrowseFilter)) + }) + return _c +} + +func (_c *SubjectRepo_Count_Call) Return(_a0 int64, _a1 error) *SubjectRepo_Count_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SubjectRepo_Count_Call) RunAndReturn(run func(context.Context, subject.BrowseFilter) (int64, error)) *SubjectRepo_Count_Call { + _c.Call.Return(run) + return _c +} + // Get provides a mock function with given fields: ctx, id, filter func (_m *SubjectRepo) Get(ctx context.Context, id uint32, filter subject.Filter) (model.Subject, error) { ret := _m.Called(ctx, id, filter) diff --git a/internal/pkg/gstr/parse.go b/internal/pkg/gstr/parse.go index 7b637447a..84287ed13 100644 --- a/internal/pkg/gstr/parse.go +++ b/internal/pkg/gstr/parse.go @@ -20,14 +20,38 @@ import ( "github.com/trim21/errgo" ) +func ParseInt8(s string) (int8, error) { + v, err := strconv.ParseInt(s, 10, 8) + + return int8(v), errgo.Wrap(err, "strconv") +} + func ParseUint8(s string) (uint8, error) { v, err := strconv.ParseUint(s, 10, 8) return uint8(v), errgo.Wrap(err, "strconv") } +func ParseUint16(s string) (uint16, error) { + v, err := strconv.ParseUint(s, 10, 16) + + return uint16(v), errgo.Wrap(err, "strconv") +} + +func ParseInt32(s string) (int32, error) { + v, err := strconv.ParseInt(s, 10, 32) + + return int32(v), errgo.Wrap(err, "strconv") +} + func ParseUint32(s string) (uint32, error) { v, err := strconv.ParseUint(s, 10, 32) return uint32(v), errgo.Wrap(err, "strconv") } + +func ParseBool(s string) (bool, error) { + v, err := strconv.ParseBool(s) + + return v, errgo.Wrap(err, "strconv") +} diff --git a/internal/subject/cache_repo.go b/internal/subject/cache_repo.go index 72ff9da52..c172a87b3 100644 --- a/internal/subject/cache_repo.go +++ b/internal/subject/cache_repo.go @@ -39,6 +39,11 @@ type cacheRepo struct { log *zap.Logger } +const ( + browseCacheTTLFirst = 24 * time.Hour + browseCacheTTLOther = time.Hour +) + func (r cacheRepo) Get(ctx context.Context, id model.SubjectID, filter Filter) (model.Subject, error) { var key = cachekey.Subject(id) @@ -71,6 +76,66 @@ func (r cacheRepo) GetByIDs( return r.repo.GetByIDs(ctx, ids, filter) } +func (r cacheRepo) Count(ctx context.Context, filter BrowseFilter) (int64, error) { + hash, err := filter.Hash() + if err != nil { + return 0, err + } + key := cachekey.SubjectBrowseCount(hash) + + var s int64 + ok, err := r.cache.Get(ctx, key, &s) + if err != nil { + return s, errgo.Wrap(err, "cache.Get") + } + if ok { + return s, nil + } + + s, err = r.repo.Count(ctx, filter) + if err != nil { + return s, err + } + if e := r.cache.Set(ctx, key, s, 10*time.Minute); e != nil { + r.log.Error("can't set response to cache", zap.Error(e)) + } + + return s, nil +} + +func (r cacheRepo) Browse( + ctx context.Context, filter BrowseFilter, limit, offset int, +) ([]model.Subject, error) { + hash, err := filter.Hash() + if err != nil { + return nil, err + } + key := cachekey.SubjectBrowse(hash, limit, offset) + + var subjects []model.Subject + ok, err := r.cache.Get(ctx, key, &subjects) + if err != nil { + return nil, errgo.Wrap(err, "cache.Get") + } + if ok { + return subjects, nil + } + + subjects, err = r.repo.Browse(ctx, filter, limit, offset) + if err != nil { + return nil, err + } + ttl := browseCacheTTLFirst + if offset > 0 { + ttl = browseCacheTTLOther + } + if e := r.cache.Set(ctx, key, subjects, ttl); e != nil { + r.log.Error("can't set response to cache", zap.Error(e)) + } + + return subjects, nil +} + func (r cacheRepo) GetPersonRelated( ctx context.Context, personID model.PersonID, ) ([]domain.SubjectPersonRelation, error) { diff --git a/internal/subject/domain.go b/internal/subject/domain.go index 2fe2c3e8e..ae176f560 100644 --- a/internal/subject/domain.go +++ b/internal/subject/domain.go @@ -16,6 +16,8 @@ package subject import ( "context" + "fmt" + "hash/fnv" "github.com/bangumi/server/domain" "github.com/bangumi/server/internal/model" @@ -27,6 +29,50 @@ type Filter struct { NSFW null.Bool } +type BrowseFilter struct { + NSFW null.Bool + Type uint8 + Category null.Uint16 + Series null.Bool + Platform null.String + Sort null.String + Year null.Int32 + Month null.Int8 +} + +func (f BrowseFilter) Hash() (string, error) { + h := fnv.New64a() + fields := []string{} + fields = append(fields, fmt.Sprintf("type:%v", f.Type)) + if f.NSFW.Set { + fields = append(fields, fmt.Sprintf("nsfw:%v", f.NSFW)) + } + if f.Category.Set { + fields = append(fields, fmt.Sprintf("category:%v", f.Category)) + } + if f.Series.Set { + fields = append(fields, fmt.Sprintf("series:%v", f.Series)) + } + if f.Platform.Set { + fields = append(fields, fmt.Sprintf("platform:%v", f.Platform)) + } + if f.Sort.Set { + fields = append(fields, fmt.Sprintf("sort:%v", f.Sort)) + } + if f.Year.Set { + fields = append(fields, fmt.Sprintf("year:%v", f.Year)) + } + if f.Month.Set { + fields = append(fields, fmt.Sprintf("month:%v", f.Month)) + } + for _, field := range fields { + if _, err := h.Write([]byte(field)); err != nil { + return "", err + } + } + return fmt.Sprintf("%x", h.Sum64()), nil +} + type Repo interface { read } @@ -40,6 +86,9 @@ type read interface { Get(ctx context.Context, id model.SubjectID, filter Filter) (model.Subject, error) GetByIDs(ctx context.Context, ids []model.SubjectID, filter Filter) (map[model.SubjectID]model.Subject, error) + Count(ctx context.Context, filter BrowseFilter) (int64, error) + Browse(ctx context.Context, filter BrowseFilter, limit, offset int) ([]model.Subject, error) + GetPersonRelated(ctx context.Context, personID model.PersonID) ([]domain.SubjectPersonRelation, error) GetCharacterRelated(ctx context.Context, characterID model.CharacterID) ([]domain.SubjectCharacterRelation, error) GetSubjectRelated(ctx context.Context, subjectID model.SubjectID) ([]domain.SubjectInternalRelation, error) diff --git a/internal/subject/mysql_repository.go b/internal/subject/mysql_repository.go index a76a9baa7..1765eb9c4 100644 --- a/internal/subject/mysql_repository.go +++ b/internal/subject/mysql_repository.go @@ -231,6 +231,94 @@ func (r mysqlRepo) GetByIDs( return result, nil } +func (r mysqlRepo) Count( + ctx context.Context, + filter BrowseFilter) (int64, error) { + q := r.q.Subject.WithContext(ctx).Joins(r.q.Subject.Fields).Join( + r.q.SubjectField, r.q.Subject.ID.EqCol(r.q.SubjectField.Sid), + ).Where(r.q.Subject.TypeID.Eq(filter.Type)) + if filter.NSFW.Set { + q = q.Where(r.q.Subject.Nsfw.Is(filter.NSFW.Value)) + } + if filter.Category.Set { + q = q.Where(r.q.Subject.Platform.Eq(filter.Category.Value)) + } + if filter.Series.Set { + q = q.Where(r.q.Subject.Series.Is(filter.Series.Value)) + } + if filter.Platform.Set { + q = q.Where(r.q.Subject.Infobox.Like(fmt.Sprintf("%%[%s]%%", filter.Platform.Value))) + } + if filter.Year.Set { + q = q.Where(r.q.SubjectField.Year.Eq(filter.Year.Value)) + } + if filter.Month.Set { + q = q.Where(r.q.SubjectField.Mon.Eq(filter.Month.Value)) + } + + if filter.Sort.Set { + switch filter.Sort.Value { + case "date": + q = q.Order(r.q.SubjectField.Date.Desc()) + case "rank": + q = q.Order(r.q.SubjectField.Rank) + } + } + + return q.Count() +} + +func (r mysqlRepo) Browse( + ctx context.Context, filter BrowseFilter, limit, offset int, +) ([]model.Subject, error) { + q := r.q.Subject.WithContext(ctx).Joins(r.q.Subject.Fields).Join( + r.q.SubjectField, r.q.Subject.ID.EqCol(r.q.SubjectField.Sid), + ).Where(r.q.Subject.TypeID.Eq(filter.Type)) + if filter.NSFW.Set { + q = q.Where(r.q.Subject.Nsfw.Is(filter.NSFW.Value)) + } + if filter.Category.Set { + q = q.Where(r.q.Subject.Platform.Eq(filter.Category.Value)) + } + if filter.Series.Set { + q = q.Where(r.q.Subject.Series.Is(filter.Series.Value)) + } + if filter.Platform.Set { + q = q.Where(r.q.Subject.Infobox.Like(fmt.Sprintf("%%[%s]%%", filter.Platform.Value))) + } + if filter.Year.Set { + q = q.Where(r.q.SubjectField.Year.Eq(filter.Year.Value)) + } + if filter.Month.Set { + q = q.Where(r.q.SubjectField.Mon.Eq(filter.Month.Value)) + } + + if filter.Sort.Set { + switch filter.Sort.Value { + case "date": + q = q.Order(r.q.SubjectField.Date.Desc()) + case "rank": + q = q.Order(r.q.SubjectField.Rank) + } + } + + subjects, err := q.Limit(limit).Offset(offset).Find() + if err != nil { + r.log.Error("unexpected error happened", zap.Error(err)) + return nil, errgo.Wrap(err, "dal") + } + + var result = make([]model.Subject, len(subjects)) + for i, subject := range subjects { + result[i], err = ConvertDao(subject) + if err != nil { + return nil, err + } + } + + return result, nil +} + func (r mysqlRepo) GetActors( ctx context.Context, subjectID model.SubjectID, diff --git a/internal/subject/mysql_repository_test.go b/internal/subject/mysql_repository_test.go index 3dfc5b4fd..adcfad4ce 100644 --- a/internal/subject/mysql_repository_test.go +++ b/internal/subject/mysql_repository_test.go @@ -62,6 +62,65 @@ func TestMysqlRepo_Get_filter(t *testing.T) { require.ErrorIs(t, err, gerr.ErrNotFound) } +func TestBrowse(t *testing.T) { + test.RequireEnv(t, test.EnvMysql) + t.Parallel() + + repo := getRepo(t) + + filter := subject.BrowseFilter{ + Type: 2, + } + s, err := repo.Browse(context.Background(), filter, 30, 0) + require.NoError(t, err) + require.Equal(t, 12, len(s)) + + filter = subject.BrowseFilter{ + Type: 1, + Category: null.New(uint16(1003)), + } + s, err = repo.Browse(context.Background(), filter, 30, 0) + require.NoError(t, err) + require.Equal(t, 2, len(s)) + + filter = subject.BrowseFilter{ + Type: 2, + Year: null.New(int32(2008)), + } + s, err = repo.Browse(context.Background(), filter, 30, 0) + require.NoError(t, err) + require.Equal(t, 2, len(s)) + + filter = subject.BrowseFilter{ + Type: 3, + Sort: null.New("rank"), + } + s, err = repo.Browse(context.Background(), filter, 30, 0) + require.NoError(t, err) + require.Equal(t, 9, len(s)) + require.Equal(t, model.SubjectID(2), s[0].ID) + require.Equal(t, model.SubjectID(20), s[1].ID) + require.Equal(t, model.SubjectID(17), s[2].ID) + require.Equal(t, model.SubjectID(16), s[3].ID) + require.Equal(t, model.SubjectID(15), s[4].ID) + require.Equal(t, model.SubjectID(406604), s[5].ID) + require.Equal(t, model.SubjectID(19), s[6].ID) + require.Equal(t, model.SubjectID(315957), s[7].ID) + require.Equal(t, model.SubjectID(18), s[8].ID) + + filter = subject.BrowseFilter{ + Type: 4, + Platform: null.New("PS3"), + Sort: null.New("date"), + } + s, err = repo.Browse(context.Background(), filter, 30, 0) + require.NoError(t, err) + require.Equal(t, 3, len(s)) + require.Equal(t, model.SubjectID(7), s[0].ID) + require.Equal(t, model.SubjectID(6), s[1].ID) + require.Equal(t, model.SubjectID(13), s[2].ID) +} + func TestMysqlRepo_GetByIDs(t *testing.T) { test.RequireEnv(t, test.EnvMysql) t.Parallel() diff --git a/openapi/components/subject_cat_anime.yaml b/openapi/components/subject_cat_anime.yaml new file mode 100644 index 000000000..42ca9c232 --- /dev/null +++ b/openapi/components/subject_cat_anime.yaml @@ -0,0 +1,31 @@ +title: SubjectAnimeCategory +example: 1 +enum: + - 0 + - 1 + - 2 + - 3 + - 5 +type: integer +description: |- + 动画类型 + - `0` 为 其他 + - `1` 为 TV + - `2` 为 OVA + - `3` 为 Movie + - `5` 为 WEB +x-ms-enum: + name: SubjectAnimeCategory + modelAsString: false + values: + - Other + - TV + - OVA + - Movie + - WEB +x-enum-varnames: + - Other + - TV + - OVA + - Movie + - WEB diff --git a/openapi/components/subject_cat_book.yaml b/openapi/components/subject_cat_book.yaml new file mode 100644 index 000000000..0c5cb8bd7 --- /dev/null +++ b/openapi/components/subject_cat_book.yaml @@ -0,0 +1,27 @@ +title: SubjectBookCategory +example: 1001 +enum: + - 0 + - 1001 + - 1002 + - 1003 +type: integer +description: |- + 书籍类型 + - `0` 为 其他 + - `1001` 为 漫画 + - `1002` 为 小说 + - `1003` 为 画集 +x-ms-enum: + name: SubjectBookCategory + modelAsString: false + values: + - Other + - Comic + - Novel + - Illustration +x-enum-varnames: + - Other + - Comic + - Novel + - Illustration diff --git a/openapi/components/subject_cat_game.yaml b/openapi/components/subject_cat_game.yaml new file mode 100644 index 000000000..ec1eeaafa --- /dev/null +++ b/openapi/components/subject_cat_game.yaml @@ -0,0 +1,31 @@ +title: SubjectGameCategory +example: 4001 +enum: + - 0 + - 4001 + - 4003 + - 4002 + - 4005 +type: integer +description: |- + 游戏类型 + - `0` 为 其他 + - `4001` 为 游戏 + - `4002` 为 软件 + - `4003` 为 扩展包 + - `4005` 为 桌游 +x-ms-enum: + name: SubjectGameCategory + modelAsString: false + values: + - Other + - Games + - Software + - DLC + - Tabletop +x-enum-varnames: + - Other + - Games + - Software + - DLC + - Tabletop diff --git a/openapi/components/subject_cat_real.yaml b/openapi/components/subject_cat_real.yaml new file mode 100644 index 000000000..317d4291b --- /dev/null +++ b/openapi/components/subject_cat_real.yaml @@ -0,0 +1,43 @@ +title: SubjectRealCategory +example: 6 +enum: + - 0 + - 1 + - 2 + - 3 + - 6001 + - 6002 + - 6003 + - 6004 +type: integer +description: |- + 电影类型 + - `0` 为 其他 + - `1` 为 日剧 + - `2` 为 欧美剧 + - `3` 为 华语剧 + - `6001` 为 电视剧 + - `6002` 为 电影 + - `6003` 为 演出 + - `6004` 为 综艺 +x-ms-enum: + name: SubjectRealCategory + modelAsString: false + values: + - Other + - JP + - EN + - CN + - TV + - Movie + - Live + - Show +x-enum-varnames: + - Other + - JP + - EN + - CN + - TV + - Movie + - Live + - Show diff --git a/openapi/components/subject_v0_slim.yaml b/openapi/components/subject_v0_slim.yaml index 8a47772c9..048929646 100644 --- a/openapi/components/subject_v0_slim.yaml +++ b/openapi/components/subject_v0_slim.yaml @@ -52,8 +52,12 @@ properties: type: integer score: description: 分数 - title: Total + title: Score type: number + rank: + description: 排名 + title: Rank + type: integer tags: description: 前 10 个 tag diff --git a/openapi/v0.yaml b/openapi/v0.yaml index 16fb81a38..efd29febe 100644 --- a/openapi/v0.yaml +++ b/openapi/v0.yaml @@ -196,6 +196,81 @@ paths: description: 排名 "type": "integer" + "/v0/subjects": + get: + tags: + - 条目 + summary: 浏览条目 + description: 第一页会 cache 24h，之后会 cache 1h + operationId: getSubjects + parameters: + - name: type + in: query + description: 条目类型 + required: true + schema: + $ref: "#/components/schemas/SubjectType" + - name: cat + in: query + description: 条目分类，参照 `SubjectCategory` enum + required: false + schema: + $ref: "#/components/schemas/SubjectCategory" + - name: series + in: query + description: 是否系列，仅对书籍类型的条目有效 + required: false + schema: + type: boolean + - name: platform + in: query + description: 平台，仅对游戏类型的条目有效 + required: false + schema: + type: string + - name: order + in: query + description: 排序，枚举值 {date|rank} + required: false + schema: + title: Sort Order + type: string + - name: year + in: query + description: 年份 + required: false + schema: + type: integer + - name: month + in: query + description: 月份 + required: false + schema: + type: integer + - $ref: "#/components/parameters/default_query_limit" + - $ref: "#/components/parameters/default_query_offset" + responses: + "200": + description: Successful Response + content: + application/json: + schema: + "$ref": "#/components/schemas/Paged_Subject" + "400": + description: Validation Error + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorDetail" + "404": + description: Not Found + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorDetail" + security: + - OptionalHTTPBearer: [] + "/v0/subjects/{subject_id}": get: tags: @@ -2310,9 +2385,7 @@ components: title: ID type: integer type: - title: Type - type: integer - description: "`0` 本篇，`1` SP，`2` OP，`3` ED" + $ref: "#/components/schemas/EpType" name: title: Name type: string @@ -2481,6 +2554,28 @@ components: Page: $ref: "./components/page.yaml" + Paged_Subject: + title: Paged[Subject] + type: object + properties: + total: + title: Total + type: integer + default: 0 + limit: + title: Limit + type: integer + default: 0 + offset: + title: Offset + type: integer + default: 0 + data: + title: Data + type: array + items: + "$ref": "#/components/schemas/Subject" + default: [] Paged_Episode: title: Paged[Episode] type: object @@ -3002,6 +3097,20 @@ components: $ref: "./components/subject_tags.yaml" SubjectType: $ref: "./components/subject_type.yaml" + SubjectBookCategory: + $ref: "./components/subject_cat_book.yaml" + SubjectAnimeCategory: + $ref: "./components/subject_cat_anime.yaml" + SubjectGameCategory: + $ref: "./components/subject_cat_game.yaml" + SubjectRealCategory: + $ref: "./components/subject_cat_real.yaml" + SubjectCategory: + anyOf: + - $ref: "#/components/schemas/SubjectBookCategory" + - $ref: "#/components/schemas/SubjectAnimeCategory" + - $ref: "#/components/schemas/SubjectGameCategory" + - $ref: "#/components/schemas/SubjectRealCategory" UserSubjectCollection: $ref: "./components/user_subject_collection.yaml" UserSubjectCollectionModifyPayload: diff --git a/readme.md b/readme.md index 91016cd9e..55104f6a9 100644 --- a/readme.md +++ b/readme.md @@ -78,13 +78,13 @@ ORM: [GORM](https://github.com/go-gorm/gorm) 和 [GORM Gen](https://github.com/g 启动 HTTP server ```shell -go run main.go --config config.yaml web +task web ``` 启动 kafka consumer ```shell -go run main.go canal --config config.yaml +task consumer ``` ### 后端环境 diff --git a/web/handler/subject/browse.go b/web/handler/subject/browse.go new file mode 100644 index 000000000..10618b137 --- /dev/null +++ b/web/handler/subject/browse.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, version 3. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see + +package subject + +import ( + "net/http" + + "github.com/labstack/echo/v4" + "github.com/trim21/errgo" + + "github.com/bangumi/server/internal/model" + "github.com/bangumi/server/internal/pkg/gstr" + "github.com/bangumi/server/internal/pkg/null" + "github.com/bangumi/server/internal/subject" + "github.com/bangumi/server/web/accessor" + "github.com/bangumi/server/web/req" + "github.com/bangumi/server/web/res" +) + +func (h Subject) Browse(c echo.Context) error { + page, err := req.GetPageQuery(c, req.DefaultPageLimit, req.DefaultMaxPageLimit) + if err != nil { + return err + } + + filter, err := parseBrowseQuery(c) + if err != nil { + return err + } + + count, err := h.subject.Count(c.Request().Context(), *filter) + if err != nil { + return errgo.Wrap(err, "failed to count subjects") + } + + if count == 0 { + return c.JSON(http.StatusOK, res.Paged{ + Data: []res.SubjectV0{}, Total: count, Limit: page.Limit, Offset: page.Offset}) + } + + if err = page.Check(count); err != nil { + return err + } + + subjects, err := h.subject.Browse(c.Request().Context(), *filter, page.Limit, page.Offset) + if err != nil { + return errgo.Wrap(err, "failed to browse subjects") + } + data := make([]res.SubjectV0, 0, len(subjects)) + for _, s := range subjects { + data = append(data, convertModelSubject(s, 0)) + } + + return c.JSON(http.StatusOK, res.Paged{Data: data, Total: count, Limit: page.Limit, Offset: page.Offset}) +} + +func parseBrowseQuery(c echo.Context) (*subject.BrowseFilter, error) { + filter := subject.BrowseFilter{} + u := accessor.GetFromCtx(c) + filter.NSFW = null.Bool{Value: !u.AllowNSFW(), Set: true} + if stype, err := req.ParseSubjectType(c.QueryParam("type")); err != nil { + return nil, res.BadRequest(err.Error()) + } else { + filter.Type = stype + } + if catStr := c.QueryParam("cat"); catStr != "" { + if cat, err := req.ParseSubjectCategory(filter.Type, catStr); err != nil { + return nil, res.BadRequest(err.Error()) + } else { + filter.Category = null.Uint16{Value: cat, Set: true} + } + } + if filter.Type == model.SubjectTypeBook { + if seriesStr := c.QueryParam("series"); seriesStr != "" { + if series, err := gstr.ParseBool(seriesStr); err != nil { + return nil, res.BadRequest(err.Error()) + } else { + filter.Series = null.Bool{Value: series, Set: true} + } + } + } + if filter.Type == model.SubjectTypeGame { + if platform := c.QueryParam("platform"); platform != "" { + // TODO: check if platform is valid + filter.Platform = null.String{Value: platform, Set: true} + } + } + if sort := c.QueryParam("sort"); sort != "" { + switch sort { + case "rank", "date": + filter.Sort = null.String{Value: sort, Set: true} + default: + return nil, res.BadRequest("unknown sort: " + sort) + } + } + if year, err := GetYearQuery(c); err != nil { + return nil, err + } else { + filter.Year = year + } + if month, err := GetMonthQuery(c); err != nil { + return nil, err + } else { + filter.Month = month + } + + return &filter, nil +} + +func GetYearQuery(c echo.Context) (null.Int32, error) { + yearStr := c.QueryParam("year") + if yearStr == "" { + return null.Int32{}, nil + } + if year, err := gstr.ParseInt32(yearStr); err != nil { + return null.Int32{}, res.BadRequest(err.Error()) + } else { + if year < 1900 || year > 3000 { + return null.Int32{}, res.BadRequest("invalid year: " + yearStr) + } + return null.Int32{Value: year, Set: true}, nil + } +} + +func GetMonthQuery(c echo.Context) (null.Int8, error) { + monthStr := c.QueryParam("month") + if monthStr == "" { + return null.Int8{}, nil + } + if month, err := gstr.ParseInt8(monthStr); err != nil { + return null.Int8{}, res.BadRequest(err.Error()) + } else { + if month < 1 || month > 12 { + return null.Int8{}, res.BadRequest("invalid month: " + monthStr) + } + return null.Int8{Value: month, Set: true}, nil + } +} diff --git a/web/handler/subject/subject.go b/web/handler/subject/subject.go index b1eec2e71..d7cbf6055 100644 --- a/web/handler/subject/subject.go +++ b/web/handler/subject/subject.go @@ -48,6 +48,7 @@ func New( } func (h *Subject) Routes(g *echo.Group) { + g.GET("/subjects", h.Browse) g.GET("/subjects/:id", h.Get) g.GET("/subjects/:id/image", h.GetImage) g.GET("/subjects/:id/persons", h.GetRelatedPersons) diff --git a/web/req/query_parse.go b/web/req/query_parse.go index e9a71c456..63a0fa085 100644 --- a/web/req/query_parse.go +++ b/web/req/query_parse.go @@ -22,6 +22,7 @@ import ( "github.com/bangumi/server/internal/model" "github.com/bangumi/server/internal/pkg/gstr" "github.com/bangumi/server/internal/pm" + "github.com/bangumi/server/pkg/vars" "github.com/bangumi/server/web/res" ) @@ -47,6 +48,24 @@ func ParseSubjectType(s string) (model.SubjectType, error) { return 0, res.BadRequest(strconv.Quote(s) + " is not a valid subject type") } +func ParseSubjectCategory(stype model.SubjectType, s string) (uint16, error) { + if s == "" { + return 0, res.BadRequest("subject category is empty") + } + platforms, ok := vars.PlatformMap[stype] + if !ok { + return 0, res.BadRequest("bad subject type: " + strconv.Quote(s)) + } + v, err := gstr.ParseUint16(s) + if err != nil { + return 0, res.BadRequest("bad subject category: " + strconv.Quote(s)) + } + if _, ok := platforms[v]; !ok { + return 0, res.BadRequest("bad subject category: " + strconv.Quote(s)) + } + return v, nil +} + func ParseID(s string) (model.CharacterID, error) { if s == "" { return 0, errMissingID