Automated cherry pick of #25711: Hotfix/qj apimap sync modelset recode (#25712)

* fix: recode apimap sync modelsets

* fix: recode apimap and vpcagent

---------

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>
This commit is contained in:
Jian QiuandQiu Jian authored and GitHub committed 2026-09-22 23:37:28 +08:00
1 parent 165f435527
commit 1eb6fb1b00
23 files changed
+1380 -480

No files matched your search

+2
View File
@@ -43,3 +43,5 @@ pkg/generated
out.gotext.json
pkg/notify/sender/email_test.go
designs
+11 -2
View File
@@ -24,8 +24,8 @@ import (
func init() {
type GetOptions struct{}
shell.R(new(GetOptions), "apimap-vpcagent", "Show net map for vpcagent", func(s *mcclient.ClientSession, _ *GetOptions) error {
ret, err := apimap.APIMap.GetVPCAgentTopo(s)
shell.R(new(GetOptions), "apimap-modelsets", "Show the model sets served by the apimap service", func(s *mcclient.ClientSession, _ *GetOptions) error {
ret, _, err := apimap.APIMap.GetModelSets(s)
if err != nil {
return err
}
@@ -33,4 +33,13 @@ func init() {
return nil
})
type StatsOptions struct{}
shell.R(new(StatsOptions), "apimap-modelsets-stats", "Show the sync status of the apimap service", func(s *mcclient.ClientSession, _ *StatsOptions) error {
ret, err := apimap.APIMap.GetModelSetsStats(s)
if err != nil {
return err
}
fmt.Print(ret.YAMLString())
return nil
})
}
+9
View File
@@ -15,7 +15,10 @@
package misc
import (
"fmt"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/modules/apimap"
"yunion.io/x/onecloud/pkg/mcclient/modules/vpcagent"
)
@@ -23,6 +26,12 @@ func init() {
type VpcAgentSyncOptions struct {
}
R(&VpcAgentSyncOptions{}, "vpcagent-sync", "Invoke sync of vpcagent", func(s *mcclient.ClientSession, args *VpcAgentSyncOptions) error {
// when the platform serves the model sets from apimap, vpcagent only
// picks them up there, so the apimap side has to be synced first
if apimap.TriggerSyncAndWait(s) == apimap.SyncOutcomeUnchanged {
fmt.Println("model sets did not change, skip vpcagent sync")
return nil
}
err := vpcagent.VpcAgent.DoSync(s)
if err != nil {
return err
+88 -20
View File
@@ -28,6 +28,8 @@ import (
api "yunion.io/x/onecloud/pkg/apis/notify"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/cloudcommon/policy"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
@@ -46,6 +48,12 @@ type APIHelper struct {
mcclientSession *mcclient.ClientSession
tick *time.Timer
// apiMapTS is the version of the model sets that modelSets reflects, as
// served by the apimap service; hasAPIMapTS tells whether that version
// is known at all. Both are only touched from the sync loop.
apiMapTS int64
hasAPIMapTS bool
}
func NewAPIHelper(opts *Options, modelSets IModelSets) (*APIHelper, error) {
@@ -76,13 +84,26 @@ func (h *APIHelper) getRunDelay() time.Duration {
func (h *APIHelper) addSyncHandler(app *appsrv.Application, prefix string) {
path := httputils.JoinPath(prefix, "sync")
app.AddHandler("POST", path, h.handlerSync)
app.AddHandler("POST", path, auth.Authenticate(RequireSystemAdmin(h.handlerSync)))
}
func (h *APIHelper) handlerSync(ctx context.Context, w http.ResponseWriter, r *http.Request) {
h.scheduleSync()
}
// RequireSystemAdmin wraps a handler so that only a caller with system scope
// privilege reaches it.
func RequireSystemAdmin(h func(context.Context, http.ResponseWriter, *http.Request)) func(context.Context, http.ResponseWriter, *http.Request) {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential)
if userCred == nil || !userCred.HasSystemAdminPrivilege() {
httperrors.ForbiddenError(ctx, w, "only system admin is allowed to access this resource")
return
}
h(ctx, w, r)
}
}
func (h *APIHelper) Start(ctx context.Context, app *appsrv.Application, prefix string) {
defer func() {
log.Infoln("apihelper: bye")
@@ -133,11 +154,11 @@ func (h *APIHelper) RunManually(ctx context.Context) {
}
func (h *APIHelper) run(ctx context.Context) {
changed, err := h.doSync(ctx)
if err != nil {
log.Errorf("doSync error: %v", err)
status := h.doSync(ctx)
if status.Err != nil {
log.Errorf("doSync error: %v", status.Err)
}
if changed {
if status.Changed {
mssCopy := h.modelSets.CopyJoined()
select {
case h.modelSetsCh <- mssCopy:
@@ -146,20 +167,35 @@ func (h *APIHelper) run(ctx context.Context) {
}
}
func (h *APIHelper) doSync(ctx context.Context) (changed bool, err error) {
{
stime := time.Now()
defer func() {
elapsed := time.Since(stime)
log.Infof("sync data done, changed: %v, elapsed: %s", changed, elapsed.String())
}()
}
func (h *APIHelper) doSync(ctx context.Context) (status SyncStatus) {
status.At = time.Now()
defer func() {
status.Elapsed = time.Since(status.At)
log.Infof("sync data done, changed: %v, elapsed: %s, err: %v", status.Changed, status.Elapsed.String(), status.Err)
if h.opts.OnSyncDone != nil {
h.opts.OnSyncDone(status)
}
}()
s := h.adminClientSession(ctx)
mss := h.modelSets.Copy()
r, err := SyncModelSets(mss, s, h.opts)
if err != nil {
return false, errors.Wrap(err, "SyncModelSets")
var (
mss IModelSets
r ModelSetsUpdateResult
err error
)
if h.opts.FetchFromComputeService {
mss = h.modelSets.Copy()
r, err = SyncModelSets(mss, s, h.opts)
if err != nil {
status.Err = errors.Wrap(err, "SyncModelSets")
return
}
} else {
mss, r, err = h.syncModelSetsFromAPIMap(s)
if err != nil {
status.Err = err
return
}
}
h.modelSets = mss
if !r.Correct {
@@ -168,10 +204,42 @@ func (h *APIHelper) doSync(ctx context.Context) (changed bool, err error) {
if err != nil {
log.Errorf("unable to EventNotify: %s", err)
}
return false, errors.Errorf("sync error")
status.Err = errors.Errorf("sync error")
return
}
changed = r.Changed
return changed, nil
status.Changed = r.Changed
status.Correct = true
return
}
// syncModelSetsFromAPIMap refreshes the model sets from the apimap service.
//
// The apimap service publishes a version with the model sets, so a round that
// finds the version unchanged is skipped without copying anything. The
// version returned along with the payload is the authoritative one: the model
// sets may have moved on between the version query and the fetch.
func (h *APIHelper) syncModelSetsFromAPIMap(s *mcclient.ClientSession) (mss IModelSets, r ModelSetsUpdateResult, err error) {
apims, ok := h.modelSets.(IAPIMapModelSets)
if !ok {
return nil, r, errors.Errorf("model sets %T cannot be fetched from apimap", h.modelSets)
}
curTS, err := apims.APIMapTimestamp(s)
if err != nil {
return nil, r, errors.Wrap(err, "APIMapTimestamp")
}
if h.hasAPIMapTS && curTS == h.apiMapTS {
// nothing new since the last round
return h.modelSets, ModelSetsUpdateResult{Correct: true, Changed: false}, nil
}
mssNews, newTS, err := apims.FetchFromAPIMap(s)
if err != nil {
return nil, r, errors.Wrap(err, "FetchFromAPIMap")
}
mss = h.modelSets.Copy()
r = mss.ApplyUpdates(mssNews)
h.apiMapTS = newTS
h.hasAPIMapTS = true
return mss, r, nil
}
func (h *APIHelper) adminClientSession(ctx context.Context) *mcclient.ClientSession {
-123
View File
@@ -1,123 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package apihelper
import (
"context"
"fmt"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/mcclient/options"
)
type GetDBModelsOptions struct {
modelOptions *GetModelsOptions
modelDBManager db.IModelManager
}
func (o GetDBModelsOptions) IncludeOtherCloudEnv() bool {
return o.modelOptions.InCludeOtherCloudEnv
}
func (o GetDBModelsOptions) GetModelSet() IModelSet {
return o.modelOptions.ModelSet
}
func (o GetDBModelsOptions) IncludeDetails() bool {
return o.modelOptions.IncludeDetails
}
func (o GetDBModelsOptions) IncludeEmulated() bool {
return o.modelOptions.IncludeEmulated
}
func (o GetDBModelsOptions) BatchListSize() int {
return o.modelOptions.BatchListSize
}
func (o GetDBModelsOptions) InCludeOtherCloudEnv() bool {
return o.modelOptions.InCludeOtherCloudEnv
}
func GetDBModels(opts *GetDBModelsOptions) error {
man := opts.modelDBManager
manKeyPlural := man.KeywordPlural()
limit := opts.BatchListSize()
// limit := 5
listOptions := options.BaseListOptions{
System: options.Bool(true),
Admin: options.Bool(true),
Scope: "system",
Details: options.Bool(opts.IncludeDetails()),
ShowEmulated: options.Bool(opts.IncludeEmulated()),
OrderBy: []string{"updated_at"},
Order: "asc",
Limit: &limit,
}
if !opts.InCludeOtherCloudEnv() {
listOptions.Filter = append(listOptions.Filter,
"manager_id.isnullorempty()", // len(manager_id) > 0 is for pubcloud objects
// "external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects
)
listOptions.CloudEnv = "onpremise"
// listOptions.Provider = []string{"OneCloud"}
}
if inter, ok := opts.GetModelSet().(IModelSetFilter); ok {
filter := inter.ModelFilter()
listOptions.Filter = append(listOptions.Filter, filter...)
}
params, err := listOptions.Params()
if err != nil {
return fmt.Errorf("%s: making list params: %s", manKeyPlural, err)
}
if inter, ok := opts.GetModelSet().(IModelListParam); ok {
filter := inter.ModelParamFilter()
params.Update(filter)
}
//XXX
//params.Set(api.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(api.LBAGENT_QUERY_ORIG_VAL))
entriesJson := []jsonutils.JSONObject{}
for {
log.Debugf("list %s with params: %s", manKeyPlural, params.String())
var err error
listResult, err := db.ListItems(man, context.Background(), auth.AdminCredential(), params, nil)
if err != nil {
return fmt.Errorf("%s: list failed: %s",
manKeyPlural, err)
}
entriesJson = append(entriesJson, listResult.Data...)
if listResult.Offset+len(listResult.Data) >= listResult.Total {
break
} else {
offset := listResult.Offset + len(listResult.Data)
params.Set("offset", jsonutils.NewInt(int64(offset)))
}
}
{
err := InitializeModelSetFromJSON(opts.GetModelSet(), entriesJson)
if err != nil {
return fmt.Errorf("%s: initializing model set failed: %s",
manKeyPlural, err)
}
}
return nil
}
+19 -61
View File
@@ -36,9 +36,19 @@ type IModelSets interface {
CopyJoined() IModelSets
}
type IDBModelSets interface {
// IAPIMapModelSets is implemented by the model sets that can be fetched from
// the apimap service instead of from the compute APIs.
type IAPIMapModelSets interface {
IModelSets
FetchFromAPIMap(s *mcclient.ClientSession) (IModelSets, error)
// APIMapTimestamp returns the version of the model sets currently served
// by the apimap service, without fetching the payload.
APIMapTimestamp(s *mcclient.ClientSession) (int64, error)
// FetchFromAPIMap fetches the model sets from the apimap service. The
// returned timestamp is the version of the returned payload, which is
// what the caller has to compare against on the next round.
FetchFromAPIMap(s *mcclient.ClientSession) (IModelSets, int64, error)
}
type IModelSet interface {
@@ -49,11 +59,6 @@ type IModelSet interface {
IncludeDetails() bool
}
type IDBModelSet interface {
IModelSet
DBModelManager() db.IModelManager
}
type IModelSetEmulatedIncluder interface {
IncludeEmulated() bool
}
@@ -70,21 +75,14 @@ type IModelListSetParams interface {
SetModelListParams(params *jsonutils.JSONDict) *jsonutils.JSONDict
}
// SyncModelSets refreshes mssOld with the model sets listed from the compute
// service APIs. Its counterpart, for the model sets served by the apimap
// service, is APIHelper's own sync path, which also has to keep track of the
// version that the model sets reflect.
func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (ModelSetsUpdateResult, error) {
var (
mssNews IModelSets
err error
)
if mssDB, ok := mssOld.(IDBModelSets); ok && !opt.FetchFromComputeService {
mssNews, err = mssDB.FetchFromAPIMap(s)
if err != nil {
return ModelSetsUpdateResult{}, errors.Wrap(err, "FetchFromAPIMap")
}
} else {
mssNews, err = syncModelSets(mssOld, s, opt)
if err != nil {
return ModelSetsUpdateResult{}, errors.Wrap(err, "syncModelSets")
}
mssNews, err := syncModelSets(mssOld, s, opt)
if err != nil {
return ModelSetsUpdateResult{}, errors.Wrap(err, "syncModelSets")
}
r := mssOld.ApplyUpdates(mssNews)
return r, nil
@@ -118,43 +116,3 @@ func syncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (
}
return mssNews, nil
}
func SyncDBModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (r ModelSetsUpdateResult, err error) {
mssNews := mssOld.NewEmpty()
for _, msNew := range mssNews.ModelSetList() {
var (
includeEmulated = false
)
if optProvider, ok := msNew.(IModelSetEmulatedIncluder); ok {
includeEmulated = optProvider.IncludeEmulated()
}
msi, ok := msNew.(IDBModelSet)
opts := &GetModelsOptions{
ClientSession: s,
ModelManager: msNew.ModelManager(),
ModelSet: msNew,
BatchListSize: opt.ListBatchSize,
IncludeDetails: msNew.IncludeDetails(),
IncludeEmulated: includeEmulated,
InCludeOtherCloudEnv: opt.IncludeOtherCloudEnv,
}
if ok {
dbOpts := &GetDBModelsOptions{
modelOptions: opts,
modelDBManager: msi.DBModelManager(),
}
err = GetDBModels(dbOpts)
if err != nil {
return
}
} else {
err = GetModels(opts)
if err != nil {
return
}
}
}
r = mssOld.ApplyUpdates(mssNews)
return r, nil
}
+25
View File
@@ -15,9 +15,30 @@
package apihelper
import (
"time"
common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
)
// SyncStatus reports the outcome of one sync attempt.
type SyncStatus struct {
// At is when the attempt started
At time.Time
// Elapsed is how long the attempt took
Elapsed time.Duration
// Changed tells whether the model sets changed in this attempt
Changed bool
// Correct tells whether the model sets are complete and consistent
Correct bool
// Err is the error of the attempt, nil on success
Err error
}
// SyncDoneFunc is called after every sync attempt, successful or not. It is
// how a service that does not consume the ModelSets() channel learns about the
// result of a sync.
type SyncDoneFunc func(status SyncStatus)
type Options struct {
common_options.CommonOptions
@@ -27,4 +48,8 @@ type Options struct {
IncludeDetails bool
IncludeOtherCloudEnv bool
FetchFromComputeService bool
// OnSyncDone, when not nil, is called after every sync attempt with the
// outcome of that attempt.
OnSyncDone SyncDoneFunc
}
-15
View File
@@ -1,15 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vpcagent // import "yunion.io/x/onecloud/pkg/apimap/models/vpcagent"
-48
View File
@@ -1,48 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vpcagent
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/apihelper"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/vpcagent/models"
)
type Result struct {
Models *models.ModelSets
Correct bool
Changed bool
}
func GetTopoResult(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (interface{}, error) {
mss := models.NewModelSets()
s := auth.GetAdminSession(ctx, "")
r, err := apihelper.SyncDBModelSets(mss, s, &apihelper.Options{
ListBatchSize: 1024,
IncludeDetails: false,
IncludeOtherCloudEnv: false,
})
ret := &Result{
Models: mss,
Correct: r.Correct,
Changed: r.Changed,
}
return ret, err
}
+28 -8
View File
@@ -17,12 +17,24 @@ package options
import (
"os"
"yunion.io/x/onecloud/pkg/apihelper"
common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
"yunion.io/x/onecloud/pkg/compute/options"
)
type SOptions struct {
options.ComputeOptions
common_options.CommonOptions
// APISyncIntervalSeconds is how often the model sets are synced from the
// region API. Keep it below the sync interval of the services that
// consume the model sets, so that a change reaches them without an extra
// delay.
APISyncIntervalSeconds int `default:"5" help:"interval in seconds to sync model sets from region"`
// APIRunDelayMilliseconds is how long a triggered sync waits before it
// runs, so that a burst of triggers collapses into one sync.
APIRunDelayMilliseconds int `default:"100" help:"delay in milliseconds before a scheduled sync runs"`
// APIListBatchSize is the batch size of the model list requests made
// against the region API.
APIListBatchSize int `default:"1024" help:"batch size of model list requests"`
}
var (
@@ -35,8 +47,21 @@ func GetOptions() *SOptions {
func Init() {
common_options.ParseOptions(&opts, os.Args, "apimap.conf", "apimap")
options.Options = opts.ComputeOptions
}
func (opts *SOptions) ValidateThenInit() error {
if opts.APISyncIntervalSeconds < apihelper.MinSyncIntervalSeconds {
opts.APISyncIntervalSeconds = apihelper.MinSyncIntervalSeconds
}
if opts.APIRunDelayMilliseconds < apihelper.MinRunDelayMilliseconds {
opts.APIRunDelayMilliseconds = apihelper.MinRunDelayMilliseconds
}
if opts.APIListBatchSize <= 20 {
opts.APIListBatchSize = 20
}
return nil
}
func OnOptionsChange(oldO, newO interface{}) bool {
oldOpts := oldO.(*SOptions)
newOpts := newO.(*SOptions)
@@ -45,11 +70,6 @@ func OnOptionsChange(oldO, newO interface{}) bool {
if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
changed = true
}
if common_options.OnDBOptionsChange(&oldOpts.DBOptions, &newOpts.DBOptions) {
changed = true
}
options.Options = newOpts.ComputeOptions
return changed
}
+343
View File
@@ -0,0 +1,343 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package service
import (
"bytes"
"compress/gzip"
"context"
"net/http"
"strconv"
"strings"
"sync"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/httputils"
"yunion.io/x/onecloud/pkg/apihelper"
"yunion.io/x/onecloud/pkg/apimap/options"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient/auth"
agentmodels "yunion.io/x/onecloud/pkg/vpcagent/models"
)
// ModelSetsService keeps the model sets of the region in memory and serves
// them to the agents (vpcagent, and later lbagent and sdnagent), so that the
// agents do not have to rebuild them from the compute APIs each on their own.
//
// Two sets are kept apart:
//
// - the staging set, held by the api helper, which is updated from the
// region API on every sync;
// - the published set, served to the agents, which is only replaced once a
// sync came back complete and consistent.
//
// The published set comes with a timestamp, a version that is bumped whenever
// the published set changes. Agents poll the version and only fetch the model
// sets when it moved, so an agent never has to guess whether what it holds is
// still current.
type ModelSetsService struct {
apih *apihelper.APIHelper
mu sync.RWMutex
// startupTS is the version the published set starts at
startupTS int64
ready bool
timestamp int64
published *agentmodels.ModelSets
serialized []byte
syncCount int64
lastSyncAt time.Time
lastElapsed time.Duration
lastChanged bool
lastCorrect bool
lastErr string
lastPublishAt time.Time
}
func NewModelSetsService(opts *options.SOptions) (*ModelSetsService, error) {
svc := &ModelSetsService{
startupTS: time.Now().Unix(),
}
apiOpts := &apihelper.Options{
CommonOptions: opts.CommonOptions,
SyncIntervalSeconds: opts.APISyncIntervalSeconds,
RunDelayMilliseconds: opts.APIRunDelayMilliseconds,
ListBatchSize: opts.APIListBatchSize,
// apimap only ever pulls from the region API
FetchFromComputeService: true,
IncludeDetails: false,
IncludeOtherCloudEnv: false,
OnSyncDone: svc.onSyncDone,
}
apih, err := apihelper.NewAPIHelper(apiOpts, agentmodels.NewModelSets())
if err != nil {
return nil, errors.Wrap(err, "new api helper")
}
svc.apih = apih
return svc, nil
}
// modelsetsPrefix is the path prefix of the model sets API. The sync endpoint
// is registered under it by the api helper, which is why the very same prefix
// is handed to APIHelper.Start.
const modelsetsPrefix = "modelsets"
func (s *ModelSetsService) InitHandlers(app *appsrv.Application) {
app.AddHandler2("GET", httputils.JoinPath("", modelsetsPrefix),
auth.Authenticate(apihelper.RequireSystemAdmin(s.handleModelSets)), nil, "get_modelsets", nil)
app.AddHandler2("GET", httputils.JoinPath(modelsetsPrefix, "timestamp"),
auth.Authenticate(apihelper.RequireSystemAdmin(s.handleModelSetsTimestamp)), nil, "get_modelsets_timestamp", nil)
app.AddHandler2("GET", httputils.JoinPath(modelsetsPrefix, "stats"),
auth.Authenticate(apihelper.RequireSystemAdmin(s.handleModelSetsStats)), nil, "get_modelsets_stats", nil)
}
func (s *ModelSetsService) Start(ctx context.Context, app *appsrv.Application) {
wg := ctx.Value("wg").(*sync.WaitGroup)
defer func() {
log.Infoln("apimap: model sets service bye")
wg.Done()
}()
wg.Add(1)
go s.apih.Start(ctx, app, modelsetsPrefix)
for {
select {
case imss := <-s.apih.ModelSets():
mss, ok := imss.(*agentmodels.ModelSets)
if !ok {
log.Errorf("apimap: unexpected model sets type %T", imss)
continue
}
s.publish(mss)
case <-ctx.Done():
return
}
}
}
// onSyncDone is called by the api helper after every sync attempt.
func (s *ModelSetsService) onSyncDone(status apihelper.SyncStatus) {
s.mu.Lock()
defer s.mu.Unlock()
s.syncCount++
s.lastSyncAt = status.At
s.lastElapsed = status.Elapsed
s.lastChanged = status.Changed
s.lastCorrect = status.Correct
if status.Err != nil {
s.lastErr = status.Err.Error()
} else {
s.lastErr = ""
}
if status.Err != nil || !status.Correct {
// the sync failed, or came back incomplete: keep serving what the
// agents already have, and keep the version as it is, so that they
// do not pick up a half baked set
log.Warningf("apimap: keep the published model sets, sync error: %v", status.Err)
return
}
if s.ready {
return
}
// at least one sync came back complete, so what we have now is worth
// serving: an empty platform is a valid answer, and the agents need one
// to tell "nothing there" from "not ready yet"
log.Infof("apimap: model sets ready")
s.ready = true
s.timestamp = s.startupTS
// nothing is handed out yet: an empty platform gets an empty payload when
// it is first asked for, while the model sets that come with the first
// successful sync become the initial version rather than a change to it
}
// publish replaces the published model sets. It is called for every change
// the api helper reports, which is what the version counts.
func (s *ModelSetsService) publish(mss *agentmodels.ModelSets) {
s.mu.Lock()
defer s.mu.Unlock()
if !s.ready {
// the api helper only reports a change once a sync came back
// complete, so this is only here to keep the two in step
s.ready = true
s.timestamp = s.startupTS
}
if s.published != nil {
// a version has already been handed out, so these model sets are a
// change to it
s.timestamp++
}
s.published = mss
s.lastPublishAt = time.Now()
s.buildPayloadLocked()
log.Infof("apimap: model sets published, timestamp: %d, stats: %s", s.timestamp, mss.Stats().Dump())
}
// buildPayloadLocked serializes the published model sets, so that a request
// only has to compress and send, rather than marshal the whole set again.
func (s *ModelSetsService) buildPayloadLocked() {
body := jsonutils.NewDict()
body.Add(jsonutils.NewInt(s.timestamp), "timestamp")
body.Add(jsonutils.Marshal(s.published), "models")
s.serialized = []byte(body.String())
}
// materializeLocked puts something in place to be handed out when the service
// is ready but no sync has published anything yet: the platform is then empty,
// and the agents have to be able to tell that from "not ready". Handing that
// empty payload out is also what pins the initial version to it, so that the
// model sets arriving later count as a change.
func (s *ModelSetsService) materializeLocked() {
if s.serialized == nil {
if s.published == nil {
s.published = agentmodels.NewModelSets()
}
s.buildPayloadLocked()
}
}
// Payload returns the serialized published model sets and their version. The
// second return value is false when the service is not ready yet.
func (s *ModelSetsService) Payload() ([]byte, bool) {
s.mu.Lock()
defer s.mu.Unlock()
if !s.ready {
return nil, false
}
s.materializeLocked()
return s.serialized, true
}
// Timestamp returns the version of the published model sets. The second
// return value is false when the service is not ready yet.
func (s *ModelSetsService) Timestamp() (int64, bool) {
s.mu.Lock()
defer s.mu.Unlock()
if !s.ready {
return 0, false
}
s.materializeLocked()
return s.timestamp, true
}
// ModelSetsStatus is what the stats endpoint reports.
type ModelSetsStatus struct {
Ready bool `json:"ready"`
Timestamp int64 `json:"timestamp"`
SyncCount int64 `json:"sync_count"`
LastSyncAt time.Time `json:"last_sync_at"`
LastSyncElapsed string `json:"last_sync_elapsed"`
LastSyncChanged bool `json:"last_sync_changed"`
LastSyncCorrect bool `json:"last_sync_correct"`
LastSyncError string `json:"last_sync_error"`
LastPublishAt time.Time `json:"last_publish_at"`
Counts *agentmodels.ModelSetsStats `json:"counts"`
}
func (s *ModelSetsService) Status() *ModelSetsStatus {
s.mu.RLock()
defer s.mu.RUnlock()
st := &ModelSetsStatus{
Ready: s.ready,
Timestamp: s.timestamp,
SyncCount: s.syncCount,
LastSyncAt: s.lastSyncAt,
LastSyncElapsed: s.lastElapsed.String(),
LastSyncChanged: s.lastChanged,
LastSyncCorrect: s.lastCorrect,
LastSyncError: s.lastErr,
LastPublishAt: s.lastPublishAt,
}
if s.published != nil {
stats := s.published.Stats()
st.Counts = &stats
}
return st
}
func (s *ModelSetsService) handleModelSets(ctx context.Context, w http.ResponseWriter, r *http.Request) {
payload, ok := s.Payload()
if !ok {
sendNotReady(ctx, w)
return
}
sendPayload(w, r, payload)
}
func (s *ModelSetsService) handleModelSetsTimestamp(ctx context.Context, w http.ResponseWriter, r *http.Request) {
ts, ok := s.Timestamp()
if !ok {
sendNotReady(ctx, w)
return
}
body := jsonutils.NewDict()
body.Add(jsonutils.NewInt(ts), "timestamp")
appsrv.SendJSON(w, body)
}
func (s *ModelSetsService) handleModelSetsStats(ctx context.Context, w http.ResponseWriter, r *http.Request) {
appsrv.SendJSON(w, jsonutils.Marshal(s.Status()))
}
func sendNotReady(ctx context.Context, w http.ResponseWriter) {
httperrors.JsonClientError(ctx, w, httputils.NewJsonClientError(
http.StatusServiceUnavailable, "ServiceUnavailable", "model sets are not ready yet"))
}
// sendPayload writes the model sets out, compressing them when the client asks
// for it. The payload is cached uncompressed, since compressing it again per
// request is much cheaper than marshalling it again.
func sendPayload(w http.ResponseWriter, r *http.Request, payload []byte) {
w.Header().Set("Content-Type", "application/json;charset=utf-8")
w.Header().Set("Vary", "Accept-Encoding")
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
if compressed, err := compressPayload(payload); err != nil {
log.Errorf("apimap: compress model sets: %v", err)
} else {
w.Header().Set("Content-Encoding", "gzip")
payload = compressed
}
}
w.Header().Set("Content-Length", strconv.Itoa(len(payload)))
w.Write(payload)
}
func compressPayload(payload []byte) ([]byte, error) {
buf := &bytes.Buffer{}
zw := gzip.NewWriter(buf)
// the payload is built once per version, so it is worth the extra bytes
// to make sure everything is written before the stream is flushed
if _, err := zw.Write(payload); err != nil {
zw.Close()
return nil, errors.Wrap(err, "write")
}
if err := zw.Close(); err != nil {
return nil, errors.Wrap(err, "close")
}
return buf.Bytes(), nil
}
+219
View File
@@ -0,0 +1,219 @@
package service
import (
"testing"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/apihelper"
agentmodels "yunion.io/x/onecloud/pkg/vpcagent/models"
)
var errTestSync = errors.Error("sync failed")
func newTestService() *ModelSetsService {
return &ModelSetsService{startupTS: 1000}
}
func modelSetsWithGuests(n int) *agentmodels.ModelSets {
mss := agentmodels.NewModelSets()
vpc := &agentmodels.Vpc{}
vpc.Id = "vpc1"
vpc.Name = "vpc1"
mss.Vpcs[vpc.Id] = vpc
for i := 0; i < n; i++ {
guest := &agentmodels.Guest{}
guest.Id = string(rune('a' + i))
guest.Name = guest.Id
guest.HostId = "host1"
mss.Guests[guest.Id] = guest
}
return mss
}
// a platform that has nothing in it is a valid answer, not a service that is
// not ready: the agents have to be able to tell the two apart
func TestEmptyPlatformBecomesReady(t *testing.T) {
s := newTestService()
if _, ok := s.Timestamp(); ok {
t.Fatal("timestamp is served before the first successful sync")
}
if _, ok := s.Payload(); ok {
t.Fatal("model sets are served before the first successful sync")
}
s.onSyncDone(apihelper.SyncStatus{Correct: true})
ts, ok := s.Timestamp()
if !ok {
t.Fatal("not ready after a successful sync")
}
if ts != 1000 {
t.Fatalf("timestamp = %d, want the startup timestamp 1000", ts)
}
payload, ok := s.Payload()
if !ok {
t.Fatal("no model sets after a successful sync")
}
obj, err := jsonutils.Parse(payload)
if err != nil {
t.Fatalf("parse payload: %v", err)
}
if _, err := obj.Get("models"); err != nil {
t.Fatalf("payload has no models: %s", payload)
}
}
// a sync that failed, or that came back incomplete, must leave both the
// published model sets and the version alone, so that the agents keep what
// they have instead of picking up a half baked set
func TestFailedSyncKeepsThePublishedVersion(t *testing.T) {
s := newTestService()
s.onSyncDone(apihelper.SyncStatus{Correct: true})
s.publish(modelSetsWithGuests(3))
ts, _ := s.Timestamp()
payload, _ := s.Payload()
if len(payload) == 0 {
t.Fatal("no payload")
}
s.onSyncDone(apihelper.SyncStatus{Err: errTestSync})
tsErr, _ := s.Timestamp()
payloadErr, _ := s.Payload()
if tsErr != ts {
t.Fatalf("timestamp moved to %d on a failed sync", tsErr)
}
if string(payloadErr) != string(payload) {
t.Fatal("payload changed on a failed sync")
}
s.onSyncDone(apihelper.SyncStatus{Correct: false})
tsBad, _ := s.Timestamp()
if tsBad != ts {
t.Fatalf("timestamp moved to %d on an incomplete sync", tsBad)
}
// and once a good sync comes back, the version moves on again
s.publish(modelSetsWithGuests(4))
tsOK, _ := s.Timestamp()
if tsOK != ts+1 {
t.Fatalf("timestamp = %d after a new version, want %d", tsOK, ts+1)
}
}
// the model sets that come with the first successful sync are the initial
// version, not a change to it; every publish after that moves the version on
func TestVersionMovesOnChanges(t *testing.T) {
s := newTestService()
s.onSyncDone(apihelper.SyncStatus{Correct: true})
s.publish(modelSetsWithGuests(1))
ts, _ := s.Timestamp()
if ts != 1000 {
t.Fatalf("first data at version %d, want the startup version 1000", ts)
}
s.publish(modelSetsWithGuests(2))
tsNext, _ := s.Timestamp()
if tsNext != ts+1 {
t.Fatalf("version %d after a change, want %d", tsNext, ts+1)
}
s.publish(modelSetsWithGuests(3))
tsLast, _ := s.Timestamp()
if tsLast != ts+2 {
t.Fatalf("version %d after another change, want %d", tsLast, ts+2)
}
}
// an empty platform is served as an empty payload, and the model sets arriving
// later have to move the version: otherwise the agents, which compare versions,
// would never come back for them
func TestVersionMovesWhenDataArrivesAfterAnEmptyStart(t *testing.T) {
s := newTestService()
s.onSyncDone(apihelper.SyncStatus{Correct: true})
tsEmpty, ok := s.Timestamp()
if !ok {
t.Fatal("an empty platform is not ready")
}
if _, ok := s.Payload(); !ok {
t.Fatal("an empty platform has no payload")
}
s.publish(modelSetsWithGuests(1))
tsData, _ := s.Timestamp()
if tsData == tsEmpty {
t.Fatalf("version stayed at %d when the data arrived, agents would not notice", tsData)
}
}
func TestStatusReportsSyncOutcome(t *testing.T) {
s := newTestService()
s.onSyncDone(apihelper.SyncStatus{Correct: true})
s.publish(modelSetsWithGuests(2))
st := s.Status()
if !st.Ready || st.SyncCount != 1 || !st.LastSyncCorrect {
t.Fatalf("unexpected status: %+v", st)
}
if st.Counts == nil || st.Counts.Guests != 2 || st.Counts.Vpcs != 1 {
t.Fatalf("unexpected counts: %+v", st.Counts)
}
// the counts travel under the same keys as the model sets themselves
dumped := jsonutils.Marshal(st.Counts)
if _, err := dumped.Get("guests"); err != nil {
t.Fatalf("counts have no guests key: %s", dumped.String())
}
}
// the payload has to be readable by the agents the very way they read it off
// the wire: parse it, take the version out of it, and rebuild the model sets
// from the "models" key
func TestPayloadIsReadableByAgents(t *testing.T) {
s := newTestService()
s.onSyncDone(apihelper.SyncStatus{Correct: true})
s.publish(modelSetsWithGuests(2))
payload, ok := s.Payload()
if !ok {
t.Fatal("no payload")
}
obj, err := jsonutils.Parse(payload)
if err != nil {
t.Fatalf("parse payload: %v", err)
}
ts, err := obj.Int("timestamp")
if err != nil {
t.Fatalf("payload has no timestamp: %v", err)
}
want, _ := s.Timestamp()
if ts != want {
t.Fatalf("payload carries version %d, serving %d", ts, want)
}
mssNews := agentmodels.NewModelSets()
if err := obj.Unmarshal(mssNews, "models"); err != nil {
t.Fatalf("unmarshal models: %v", err)
}
if len(mssNews.Guests) != 2 || len(mssNews.Vpcs) != 1 {
t.Fatalf("agents would see %s", mssNews.Stats().Dump())
}
}
func TestCompressPayload(t *testing.T) {
payload := []byte(`{"timestamp":1,"models":{}}`)
compressed, err := compressPayload(payload)
if err != nil {
t.Fatalf("compress: %v", err)
}
if len(compressed) == 0 {
t.Fatal("empty compressed payload")
}
if string(compressed) == string(payload) {
t.Fatal("payload was not compressed")
}
}
+42 -34
View File
@@ -16,55 +16,63 @@ package service
import (
"context"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/appctx"
"yunion.io/x/onecloud/pkg/apimap/models/vpcagent"
"yunion.io/x/onecloud/pkg/apimap/options"
compute_api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/apis"
app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
"yunion.io/x/onecloud/pkg/cloudcommon/policy"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/scheduler/service"
)
func StartService() {
options.Init()
opt := options.GetOptions()
if err := opt.ValidateThenInit(); err != nil {
log.Fatalf("validate options: %v", err)
}
// hack: set GuestManager recordChecksum to false
models.GuestManager.SetEnableRecordChecksum(false)
service.StartServiceWrapper(&opt.DBOptions, &opt.CommonOptions, func(app *appsrv.Application) error {
common_options.StartOptionManager(&opt, opt.ConfigSyncPeriodSeconds, compute_api.SERVICE_TYPE, compute_api.SERVICE_VERSION, options.OnOptionsChange)
InitHandlers(app)
app_common.ServeForever(app, &opt.BaseOptions)
return nil
commonOpts := &opt.CommonOptions
app_common.InitAuth(commonOpts, func() {
log.Infof("auth finished ok")
})
}
common_options.StartOptionManager(opt, opt.ConfigSyncPeriodSeconds, apis.SERVICE_TYPE_APIMAP, "", options.OnOptionsChange)
func InitHandlers(app *appsrv.Application) {
app_common.ExportOptionsHandler(app, options.GetOptions())
app.AddHandler2("GET", "/vpcagent", auth.Authenticate(vpcAgentHandler), nil, "get_vpcagent_topo", nil)
}
app := app_common.InitApp(&opt.BaseOptions, false)
app_common.ExportOptionsHandler(app, opt)
func vpcAgentHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential)
query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
svc, err := NewModelSetsService(opt)
if err != nil {
httperrors.GeneralServerError(ctx, w, err)
return
log.Fatalf("new model sets service: %v", err)
}
svc.InitHandlers(app)
result, err := vpcagent.GetTopoResult(ctx, userCred, query)
if err != nil {
httperrors.GeneralServerError(ctx, w, err)
return
}
appsrv.SendJSON(w, jsonutils.Marshal(result))
go func() {
ctx := context.Background()
ctx, cancelFunc := context.WithCancel(ctx)
wg := &sync.WaitGroup{}
ctx = context.WithValue(ctx, "wg", wg)
ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "apimap")
wg.Add(1)
go svc.Start(ctx, app)
go func() {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT)
signal.Notify(sigChan, syscall.SIGTERM)
sig := <-sigChan
log.Infof("signal received: %s", sig)
cancelFunc()
}()
wg.Wait()
}()
app_common.ServeForeverWithCleanup(app, &opt.BaseOptions, nil)
}
+7 -3
View File
@@ -27,6 +27,7 @@ import (
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/mcclient/modules/apimap"
"yunion.io/x/onecloud/pkg/mcclient/modules/vpcagent"
"yunion.io/x/onecloud/pkg/util/logclient"
)
@@ -145,9 +146,12 @@ func (task *GuestStartTask) OnStartComplete(ctx context.Context, obj db.IStandal
log.Errorf("IsOneCloudVpcNetwork fail: %s", err)
} else if isVpc {
// force update VPC topo
err := vpcagent.VpcAgent.DoSync(auth.GetAdminSession(ctx, options.Options.Region))
if err != nil {
log.Errorf("vpcagent.VpcAgent.DoSync fail %s", err)
session := auth.GetAdminSession(ctx, options.Options.Region)
if apimap.TriggerSyncAndWait(session).ShouldTriggerAgent() {
err := vpcagent.VpcAgent.DoSync(session)
if err != nil {
log.Errorf("vpcagent.VpcAgent.DoSync fail %s", err)
}
}
}
// log
+9 -5
View File
@@ -28,6 +28,7 @@ import (
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/mcclient/modules/apimap"
"yunion.io/x/onecloud/pkg/mcclient/modules/vpcagent"
"yunion.io/x/onecloud/pkg/util/logclient"
)
@@ -117,12 +118,15 @@ func (self *GuestSyncConfTask) StartRestartNetworkTask(ctx context.Context, gues
}
if isVpcNetwork {
err = vpcagent.VpcAgent.DoSync(auth.GetAdminSession(ctx, options.Options.Region))
if err != nil {
log.Errorf("vpcagent.VpcAgent.DoSync fail %s", err)
session := auth.GetAdminSession(ctx, options.Options.Region)
if apimap.TriggerSyncAndWait(session).ShouldTriggerAgent() {
err = vpcagent.VpcAgent.DoSync(session)
if err != nil {
log.Errorf("vpcagent.VpcAgent.DoSync fail %s", err)
}
// wait for vpcagent sync network topo
time.Sleep(10 * time.Second)
}
// wait for vpcagent sync network topo
time.Sleep(10 * time.Second)
}
return guest.StartQgaRestartNetworkTask(ctx, self.UserCred, "", ifnameDevice, ipMask, gateway, ip6Mask, gateway6)
}()
+151
View File
@@ -0,0 +1,151 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package apimap
import (
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/modulebase"
"yunion.io/x/onecloud/pkg/mcclient/modules"
)
var (
APIMap APIMapManager
)
func init() {
APIMap = APIMapManager{
ResourceManager: modules.NewAPIMapManager("", "", nil, nil),
}
}
type APIMapManager struct {
modulebase.ResourceManager
}
// GetModelSets returns the model sets served by the apimap service, along with
// the version they correspond to.
func (m APIMapManager) GetModelSets(s *mcclient.ClientSession) (jsonutils.JSONObject, int64, error) {
_, ret, err := modulebase.JsonRequest(m.ResourceManager, s, "GET", "/modelsets", nil, nil)
if err != nil {
return nil, 0, err
}
ts, err := ret.Int("timestamp")
if err != nil {
return nil, 0, errors.Wrap(err, "timestamp")
}
return ret, ts, nil
}
// GetModelSetsTimestamp returns the version of the model sets currently served
// by the apimap service, without fetching the model sets themselves.
func (m APIMapManager) GetModelSetsTimestamp(s *mcclient.ClientSession) (int64, error) {
_, ret, err := modulebase.JsonRequest(m.ResourceManager, s, "GET", "/modelsets/timestamp", nil, nil)
if err != nil {
return 0, err
}
return ret.Int("timestamp")
}
// GetModelSetsStats returns the sync status and the model set sizes of the
// apimap service.
func (m APIMapManager) GetModelSetsStats(s *mcclient.ClientSession) (jsonutils.JSONObject, error) {
_, ret, err := modulebase.JsonRequest(m.ResourceManager, s, "GET", "/modelsets/stats", nil, nil)
return ret, err
}
// SyncModelSets asks the apimap service to sync its model sets. The sync runs
// asynchronously on the apimap side: the call returns as soon as the sync has
// been scheduled, which is why a caller that needs the new data to be there
// uses TriggerSyncAndWait instead.
func (m APIMapManager) SyncModelSets(s *mcclient.ClientSession) error {
_, _, err := modulebase.JsonRequest(m.ResourceManager, s, "POST", "/modelsets/sync", nil, nil)
return err
}
const (
// syncPollInterval is how often the model sets version is polled while
// waiting for a sync to complete
syncPollInterval = time.Second
// syncWaitTimeout is how long to wait for the model sets to change
syncWaitTimeout = 10 * time.Second
)
// SyncOutcome is what TriggerSyncAndWait made of the apimap service.
type SyncOutcome int
const (
// SyncOutcomeUnavailable means the apimap service could not be reached at
// all, typically because it is not deployed. A caller then carries on
// the way it did before apimap existed.
SyncOutcomeUnavailable SyncOutcome = iota
// SyncOutcomeChanged means apimap published a new version of the model
// sets: the agents that consume them have something to fetch.
SyncOutcomeChanged
// SyncOutcomeUnchanged means apimap is there but published nothing new
// within the timeout. "The data did not change" and "the sync is still
// running" look the same from here, so this is a normal outcome rather
// than an error.
SyncOutcomeUnchanged
)
// ShouldTriggerAgent reports whether the caller should go on and trigger the
// agent that consumes the model sets (vpcagent, and later lbagent and
// sdnagent). There is nothing to fetch when apimap is there and published
// nothing new, and the caller stops there.
func (o SyncOutcome) ShouldTriggerAgent() bool {
return o != SyncOutcomeUnchanged
}
// TriggerSyncAndWait triggers a model set sync of the apimap service and waits
// until the model sets change, so that a caller that goes on to trigger a
// downstream agent knows that the new data is there to be fetched.
//
// It is best effort on purpose, because the apimap service is optional and a
// caller must never fail a task such as starting a server because of it: an
// endpoint that does not resolve, a service that is not ready yet or a failed
// call are logged and reported as SyncOutcomeUnavailable, which leaves the
// caller doing what it did before apimap existed.
func TriggerSyncAndWait(s *mcclient.ClientSession) SyncOutcome {
t0, err := APIMap.GetModelSetsTimestamp(s)
if err != nil {
log.Warningf("apimap: cannot read model sets timestamp: %v", err)
return SyncOutcomeUnavailable
}
err = APIMap.SyncModelSets(s)
if err != nil {
log.Warningf("apimap: cannot trigger model sets sync: %v", err)
return SyncOutcomeUnavailable
}
deadline := time.Now().Add(syncWaitTimeout)
for time.Now().Before(deadline) {
time.Sleep(syncPollInterval)
ts, err := APIMap.GetModelSetsTimestamp(s)
if err != nil {
log.Warningf("apimap: cannot read model sets timestamp: %v", err)
return SyncOutcomeUnavailable
}
if ts != t0 {
return SyncOutcomeChanged
}
}
log.Infof("apimap: model sets did not change within %s", syncWaitTimeout)
return SyncOutcomeUnchanged
}
-42
View File
@@ -1,42 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package apimap
import (
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/modulebase"
"yunion.io/x/onecloud/pkg/mcclient/modules"
)
var (
APIMap APIMapManager
)
func init() {
APIMap = APIMapManager{
ResourceManager: modules.NewAPIMapManager("", "", nil, nil),
}
}
type APIMapManager struct {
modulebase.ResourceManager
}
func (m APIMapManager) GetVPCAgentTopo(s *mcclient.ClientSession) (jsonutils.JSONObject, error) {
_, ret, err := modulebase.JsonRequest(m.ResourceManager, s, "GET", "/vpcagent", nil, nil)
return ret, err
}
+4 -4
View File
@@ -42,7 +42,7 @@ func (el *Vpc) Copy() *Vpc {
type RouteTable struct {
compute_models.SRouteTable
Vpc *Vpc
Vpc *Vpc `json:"-"`
}
func (el *RouteTable) Copy() *RouteTable {
@@ -54,7 +54,7 @@ func (el *RouteTable) Copy() *RouteTable {
type Wire struct {
compute_models.SWire
Vpc *Vpc
Vpc *Vpc `json:"-"`
}
func (el *Wire) Copy() *Wire {
@@ -263,7 +263,7 @@ func (el *Elasticip) Copy() *Elasticip {
type DnsRecord struct {
compute_models.SDnsRecord
DnsZone *DnsZone
DnsZone *DnsZone `json:"-"`
}
func (el *DnsRecord) Copy() *DnsRecord {
@@ -275,7 +275,7 @@ func (el *DnsRecord) Copy() *DnsRecord {
type DnsZone struct {
compute_models.SDnsZone
Records DnsRecords
Records DnsRecords `json:"-"`
}
func (el *DnsZone) Copy() *DnsZone {
-85
View File
@@ -23,7 +23,6 @@ import (
"yunion.io/x/onecloud/pkg/apihelper"
computeapis "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/compute/models"
mcclient_modulebase "yunion.io/x/onecloud/pkg/mcclient/modulebase"
mcclient_modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
)
@@ -61,10 +60,6 @@ func (set Vpcs) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.Vpcs
}
func (set Vpcs) DBModelManager() db.IModelManager {
return models.VpcManager
}
func (set Vpcs) NewModel() db.IModel {
return &Vpc{}
}
@@ -171,10 +166,6 @@ func (set Wires) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.Wires
}
func (set Wires) DBModelManager() db.IModelManager {
return models.WireManager
}
func (set Wires) NewModel() db.IModel {
return &Wire{}
}
@@ -224,10 +215,6 @@ func (set Guests) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.Servers
}
func (set Guests) DBModelManager() db.IModelManager {
return models.GuestManager
}
func (set Guests) NewModel() db.IModel {
return &Guest{}
}
@@ -343,10 +330,6 @@ func (set Hosts) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.Hosts
}
func (set Hosts) DBModelManager() db.IModelManager {
return models.HostManager
}
func (set Hosts) NewModel() db.IModel {
return &Host{}
}
@@ -378,10 +361,6 @@ func (set Networks) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.Networks
}
func (set Networks) DBModelManager() db.IModelManager {
return models.NetworkManager
}
func (set Networks) ModelParamFilter() jsonutils.JSONObject {
params := jsonutils.NewDict()
params.Add(jsonutils.NewString("OneCloud"), "provider")
@@ -504,10 +483,6 @@ func (set Guestnetworks) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.Servernetworks
}
func (set Guestnetworks) DBModelManager() db.IModelManager {
return models.GuestnetworkManager
}
func (set Guestnetworks) NewModel() db.IModel {
return &Guestnetwork{}
}
@@ -628,10 +603,6 @@ func (set Guestnetworksecgroups) ModelManager() mcclient_modulebase.IBaseManager
return &mcclient_modules.Servernetworksecgroups
}
func (set Guestnetworksecgroups) DBModelManager() db.IModelManager {
return models.GuestnetworksecgroupManager
}
func (set Guestnetworksecgroups) NewModel() db.IModel {
return &Guestnetworksecgroup{}
}
@@ -670,10 +641,6 @@ func (set NetworkAddresses) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.NetworkAddresses
}
func (set NetworkAddresses) DBModelManager() db.IModelManager {
return models.NetworkAddressManager
}
func (set NetworkAddresses) NewModel() db.IModel {
return &NetworkAddress{}
}
@@ -699,10 +666,6 @@ func (set SecurityGroups) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.SecGroups
}
func (set SecurityGroups) DBModelManager() db.IModelManager {
return models.SecurityGroupManager
}
func (set SecurityGroups) NewModel() db.IModel {
return &SecurityGroup{}
}
@@ -755,10 +718,6 @@ func (set SecurityGroupRules) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.SecGroupRules
}
func (set SecurityGroupRules) DBModelManager() db.IModelManager {
return models.SecurityGroupRuleManager
}
func (set SecurityGroupRules) NewModel() db.IModel {
return &SecurityGroupRule{}
}
@@ -784,10 +743,6 @@ func (set Guestsecgroups) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.Serversecgroups
}
func (set Guestsecgroups) DBModelManager() db.IModelManager {
return models.GuestsecgroupManager
}
func (set Guestsecgroups) NewModel() db.IModel {
return &Guestsecgroup{}
}
@@ -854,10 +809,6 @@ func (set Elasticips) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.Elasticips
}
func (set Elasticips) DBModelManager() db.IModelManager {
return models.ElasticipManager
}
func (set Elasticips) NewModel() db.IModel {
return &Elasticip{}
}
@@ -887,10 +838,6 @@ func (set DnsZones) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.DnsZones
}
func (set DnsZones) DBModelManager() db.IModelManager {
return models.DnsZoneManager
}
func (set DnsZones) NewModel() db.IModel {
return &DnsZone{}
}
@@ -931,10 +878,6 @@ func (set DnsRecords) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.DnsRecords
}
func (set DnsRecords) DBModelManager() db.IModelManager {
return models.DnsRecordManager
}
func (set DnsRecords) NewModel() db.IModel {
return &DnsRecord{}
}
@@ -960,10 +903,6 @@ func (set RouteTables) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.RouteTables
}
func (set RouteTables) DBModelManager() db.IModelManager {
return models.RouteTableManager
}
func (set RouteTables) NewModel() db.IModel {
return &RouteTable{}
}
@@ -989,10 +928,6 @@ func (set Groupguests) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.InstanceGroupGuests
}
func (set Groupguests) DBModelManager() db.IModelManager {
return models.GroupguestManager
}
func (set Groupguests) NewModel() db.IModel {
return &Groupguest{}
}
@@ -1019,10 +954,6 @@ func (set LoadbalancerNetworks) ModelManager() mcclient_modulebase.IBaseManager
return &mcclient_modules.Loadbalancernetworks
}
func (set LoadbalancerNetworks) DBModelManager() db.IModelManager {
return models.LoadbalancernetworkManager
}
func (set LoadbalancerNetworks) NewModel() db.IModel {
return &LoadbalancerNetwork{}
}
@@ -1049,10 +980,6 @@ func (set Groupnetworks) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.InstancegroupNetworks
}
func (set Groupnetworks) DBModelManager() db.IModelManager {
return models.GroupnetworkManager
}
func (set Groupnetworks) NewModel() db.IModel {
return &Groupnetwork{}
}
@@ -1106,10 +1033,6 @@ func (set Groups) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.InstanceGroups
}
func (set Groups) DBModelManager() db.IModelManager {
return models.GroupManager
}
func (set Groups) NewModel() db.IModel {
return &Group{}
}
@@ -1210,10 +1133,6 @@ func (set LoadbalancerListeners) ModelManager() mcclient_modulebase.IBaseManager
return &mcclient_modules.LoadbalancerListeners
}
func (set LoadbalancerListeners) DBModelManager() db.IModelManager {
return models.LoadbalancerListenerManager
}
func (set LoadbalancerListeners) NewModel() db.IModel {
return &LoadbalancerListener{}
}
@@ -1254,10 +1173,6 @@ func (set LoadbalancerAcls) ModelManager() mcclient_modulebase.IBaseManager {
return &mcclient_modules.LoadbalancerAcls
}
func (set LoadbalancerAcls) DBModelManager() db.IModelManager {
return models.LoadbalancerAclManager
}
func (set LoadbalancerAcls) NewModel() db.IModel {
return &LoadbalancerAcl{}
}
+122 -27
View File
@@ -18,6 +18,7 @@ import (
"strings"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
@@ -82,32 +83,66 @@ func NewModelSetsMaxUpdatedAt() *ModelSetsMaxUpdatedAt {
}
}
// The json tags below pin the wire format. ModelSets is the payload of the
// apimap /modelsets API, and its keys are derived from the field names when no
// tag is set. A rename would silently change the key, and Unmarshal drops an
// unmatched key without an error, so a whole model set would go missing
// unnoticed. Keep these tags in sync with expectedModelSetKeys in
// modelsets_wire_test.go.
type ModelSetsStats struct {
Vpcs int `json:"vpcs"`
Wires int `json:"wires"`
Networks int `json:"networks"`
Guests int `json:"guests"`
Hosts int `json:"hosts"`
SecurityGroups int `json:"security_groups"`
SecurityGroupRules int `json:"security_group_rules"`
Guestnetworks int `json:"guestnetworks"`
Guestsecgroups int `json:"guestsecgroups"`
Elasticips int `json:"elasticips"`
NetworkAddresses int `json:"network_addresses"`
Guestnetworksecgroups int `json:"guestnetworksecgroups"`
DnsZones int `json:"dns_zones"`
DnsRecords int `json:"dns_records"`
RouteTables int `json:"route_tables"`
Groupguests int `json:"groupguests"`
Groupnetworks int `json:"groupnetworks"`
Groups int `json:"groups"`
LoadbalancerNetworks int `json:"loadbalancer_networks"`
LoadbalancerListeners int `json:"loadbalancer_listeners"`
LoadbalancerAcls int `json:"loadbalancer_acls"`
}
type ModelSets struct {
Vpcs Vpcs
Wires Wires
Networks Networks
Guests Guests
Hosts Hosts
SecurityGroups SecurityGroups
SecurityGroupRules SecurityGroupRules
Guestnetworks Guestnetworks
Guestsecgroups Guestsecgroups
Elasticips Elasticips
NetworkAddresses NetworkAddresses
Guestnetworksecgroups Guestnetworksecgroups
Vpcs Vpcs `json:"vpcs"`
Wires Wires `json:"wires"`
Networks Networks `json:"networks"`
Guests Guests `json:"guests"`
Hosts Hosts `json:"hosts"`
SecurityGroups SecurityGroups `json:"security_groups"`
SecurityGroupRules SecurityGroupRules `json:"security_group_rules"`
Guestnetworks Guestnetworks `json:"guestnetworks"`
Guestsecgroups Guestsecgroups `json:"guestsecgroups"`
Elasticips Elasticips `json:"elasticips"`
NetworkAddresses NetworkAddresses `json:"network_addresses"`
Guestnetworksecgroups Guestnetworksecgroups `json:"guestnetworksecgroups"`
DnsZones DnsZones
DnsRecords DnsRecords
DnsZones DnsZones `json:"dns_zones"`
DnsRecords DnsRecords `json:"dns_records"`
RouteTables RouteTables
RouteTables RouteTables `json:"route_tables"`
Groupguests Groupguests
Groupnetworks Groupnetworks
Groups Groups
Groupguests Groupguests `json:"groupguests"`
Groupnetworks Groupnetworks `json:"groupnetworks"`
Groups Groups `json:"groups"`
LoadbalancerNetworks LoadbalancerNetworks
LoadbalancerListeners LoadbalancerListeners
LoadbalancerAcls LoadbalancerAcls
LoadbalancerNetworks LoadbalancerNetworks `json:"loadbalancer_networks"`
LoadbalancerListeners LoadbalancerListeners `json:"loadbalancer_listeners"`
LoadbalancerAcls LoadbalancerAcls `json:"loadbalancer_acls"`
}
func NewModelSets() *ModelSets {
@@ -236,16 +271,31 @@ func (mss *ModelSets) ApplyUpdates(mssNews apihelper.IModelSets) apihelper.Model
return r
}
func (mss *ModelSets) FetchFromAPIMap(s *mcclient.ClientSession) (apihelper.IModelSets, error) {
mssNews := mss.NewEmpty()
ret, err := apimap.APIMap.GetVPCAgentTopo(s)
var (
_ apihelper.IModelSets = (*ModelSets)(nil)
_ apihelper.IAPIMapModelSets = (*ModelSets)(nil)
)
// APIMapTimestamp returns the version of the model sets currently served by
// the apimap service.
func (mss *ModelSets) APIMapTimestamp(s *mcclient.ClientSession) (int64, error) {
return apimap.APIMap.GetModelSetsTimestamp(s)
}
// FetchFromAPIMap fetches the model sets from the apimap service, along with
// the version of the payload. The payload is flat: the join links between
// models are all tagged json:"-", so they are rebuilt by ApplyUpdates and
// join() rather than carried over the wire.
func (mss *ModelSets) FetchFromAPIMap(s *mcclient.ClientSession) (apihelper.IModelSets, int64, error) {
ret, ts, err := apimap.APIMap.GetModelSets(s)
if err != nil {
return nil, errors.Wrap(err, "GetVPCAgentTopo")
return nil, 0, errors.Wrap(err, "GetModelSets")
}
mssNews := mss.NewEmpty()
if err := ret.Unmarshal(mssNews, "models"); err != nil {
return nil, errors.Wrap(err, "Unmarshal topo")
return nil, 0, errors.Wrap(err, "unmarshal models")
}
return mssNews, nil
return mssNews, ts, nil
}
func (mss *ModelSets) join() bool {
@@ -318,3 +368,48 @@ func (mss *ModelSets) join() bool {
}
return ret
}
func (mss ModelSets) Stats() ModelSetsStats {
return ModelSetsStats{
Vpcs: len(mss.Vpcs),
Wires: len(mss.Wires),
Networks: len(mss.Networks),
Guests: len(mss.Guests),
Hosts: len(mss.Hosts),
SecurityGroups: len(mss.SecurityGroups),
SecurityGroupRules: len(mss.SecurityGroupRules),
Guestnetworks: len(mss.Guestnetworks),
Guestsecgroups: len(mss.Guestsecgroups),
Elasticips: len(mss.Elasticips),
NetworkAddresses: len(mss.NetworkAddresses),
Guestnetworksecgroups: len(mss.Guestnetworksecgroups),
DnsZones: len(mss.DnsZones),
DnsRecords: len(mss.DnsRecords),
RouteTables: len(mss.RouteTables),
Groupguests: len(mss.Groupguests),
Groupnetworks: len(mss.Groupnetworks),
Groups: len(mss.Groups),
LoadbalancerNetworks: len(mss.LoadbalancerNetworks),
LoadbalancerListeners: len(mss.LoadbalancerListeners),
LoadbalancerAcls: len(mss.LoadbalancerAcls),
}
}
// Total is the number of models in all the model sets, which is what a worker
// that pushes them out compares batches by.
func (msss ModelSetsStats) Total() int {
return msss.Vpcs + msss.Wires + msss.Networks + msss.Guests + msss.Hosts +
msss.SecurityGroups + msss.SecurityGroupRules + msss.Guestnetworks +
msss.Guestsecgroups + msss.Elasticips + msss.NetworkAddresses +
msss.Guestnetworksecgroups + msss.DnsZones + msss.DnsRecords +
msss.RouteTables + msss.Groupguests + msss.Groupnetworks + msss.Groups +
msss.LoadbalancerNetworks + msss.LoadbalancerListeners + msss.LoadbalancerAcls
}
func (msss ModelSetsStats) Dump() string {
return jsonutils.Marshal(msss).String()
}
+253
View File
@@ -0,0 +1,253 @@
package models
import (
"reflect"
"regexp"
"sort"
"strings"
"testing"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/util/reflectutils"
)
var nodeRefPattern = regexp.MustCompile(`<[0-9]+>`)
// expectedModelSetKeys pins the wire format of ModelSets: the json key of every
// model set, sorted. Marshal and Unmarshal both derive the key through
// reflectutils.Info.MarshalName(), and Unmarshal silently drops an unmatched
// key, so a renamed field would drop a whole model set without an error. Keep
// this list in sync with the json tags of ModelSets.
var expectedModelSetKeys = []string{
"dns_records",
"dns_zones",
"elasticips",
"groupguests",
"groupnetworks",
"groups",
"guestnetworks",
"guestnetworksecgroups",
"guests",
"guestsecgroups",
"hosts",
"loadbalancer_acls",
"loadbalancer_listeners",
"loadbalancer_networks",
"network_addresses",
"networks",
"route_tables",
"security_group_rules",
"security_groups",
"vpcs",
"wires",
}
func buildModelSetsForWireTest() *ModelSets {
mss := NewModelSets()
vpc := &Vpc{}
vpc.Id = "vpc1"
vpc.Name = "vpc1"
mss.Vpcs[vpc.Id] = vpc
wire := &Wire{}
wire.Id = "wire1"
wire.Name = "wire1"
wire.VpcId = "vpc1"
mss.Wires[wire.Id] = wire
rt := &RouteTable{}
rt.Id = "rt1"
rt.Name = "rt1"
rt.VpcId = "vpc1"
mss.RouteTables[rt.Id] = rt
net := &Network{}
net.Id = "net1"
net.Name = "net1"
net.WireId = "wire1"
mss.Networks[net.Id] = net
zone := &DnsZone{}
zone.Id = "zone1"
zone.Name = "zone1"
mss.DnsZones[zone.Id] = zone
rec := &DnsRecord{}
rec.Id = "rec1"
rec.Name = "rec1"
rec.DnsZoneId = "zone1"
mss.DnsRecords[rec.Id] = rec
host := &Host{}
host.Id = "host1"
host.Name = "host1"
mss.Hosts[host.Id] = host
guest := &Guest{}
guest.Id = "guest1"
guest.Name = "guest1"
guest.HostId = "host1"
mss.Guests[guest.Id] = guest
gn := &Guestnetwork{}
gn.RowId = 1
gn.GuestId = "guest1"
gn.NetworkId = "net1"
gn.IpAddr = "10.0.0.2"
mss.Guestnetworks["1"] = gn
return mss
}
// TestModelSetsKeyList checks the wire key of every model set, and that each key
// is matched back to exactly one model set by Unmarshal.
func TestModelSetsKeyList(t *testing.T) {
fields := reflectutils.FetchStructFieldValueSet(reflect.ValueOf(ModelSets{}))
keys := make([]string, 0, len(fields))
for i := range fields {
keys = append(keys, fields[i].Info.MarshalName())
}
sort.Strings(keys)
if !reflect.DeepEqual(keys, expectedModelSetKeys) {
t.Fatalf("model set keys changed:\n got: %v\nwant: %v", keys, expectedModelSetKeys)
}
}
// TestModelSetsStatsKeyList checks that ModelSetsStats carries the same wire
// keys as ModelSets, so that /modelsets/stats stays consistent with the model
// sets it counts.
func TestModelSetsStatsKeyList(t *testing.T) {
fields := reflectutils.FetchStructFieldValueSet(reflect.ValueOf(ModelSetsStats{}))
keys := make([]string, 0, len(fields))
for i := range fields {
keys = append(keys, fields[i].Info.MarshalName())
}
sort.Strings(keys)
if !reflect.DeepEqual(keys, expectedModelSetKeys) {
t.Fatalf("model sets stats keys changed:\n got: %v\nwant: %v", keys, expectedModelSetKeys)
}
}
func TestModelSetsKeysMatchedByUnmarshal(t *testing.T) {
for _, k := range expectedModelSetKeys {
js := `{"` + k + `":{"probe":{"id":"probe","name":"probe"}}}`
obj, err := jsonutils.Parse([]byte(js))
if err != nil {
t.Fatalf("parse probe for key %s: %v", k, err)
}
mss := NewModelSets()
if err := obj.Unmarshal(mss); err != nil {
t.Fatalf("unmarshal probe for key %s: %v", k, err)
}
matched := 0
for _, ms := range mss.ModelSetList() {
if reflect.ValueOf(ms).Len() > 0 {
matched++
}
}
if matched != 1 {
t.Fatalf("key %q matched %d model sets, want exactly 1", k, matched)
}
}
}
// TestModelSetsFlatRoundTrip joins a ModelSets, checks that the marshaled form
// is flat (no join link, no node reference, so that a plain Parse can read it
// back), and that the receiver side restores the links through ApplyUpdates.
func TestModelSetsFlatRoundTrip(t *testing.T) {
mss := buildModelSetsForWireTest()
mss.join()
if mss.Wires["wire1"].Vpc == nil {
t.Fatal("join did not link wire->vpc")
}
if mss.RouteTables["rt1"].Vpc == nil {
t.Fatal("join did not link routetable->vpc")
}
if mss.DnsRecords["rec1"].DnsZone == nil {
t.Fatal("join did not link dnsrecord->dnszone")
}
if mss.Networks["net1"].Wire == nil {
t.Fatal("join did not link network->wire")
}
s := jsonutils.Marshal(mss).String()
if strings.Contains(s, "___jnid_") {
t.Fatalf("marshaled output contains node id marker ___jnid_")
}
if nodeRefPattern.MatchString(s) {
t.Fatalf("marshaled output contains a node reference <N>")
}
obj, err := jsonutils.Parse([]byte(s))
if err != nil {
t.Fatalf("plain Parse of marshaled modelsets failed: %v", err)
}
// flatness: no join link carried over the wire
for _, c := range []struct {
path []string
keys []string
}{
{[]string{"wires", "wire1"}, []string{"vpc", "Vpc"}},
{[]string{"route_tables", "rt1"}, []string{"vpc", "Vpc"}},
{[]string{"dns_records", "rec1"}, []string{"dns_zone", "DnsZone"}},
{[]string{"dns_zones", "zone1"}, []string{"records", "Records"}},
{[]string{"networks", "net1"}, []string{"wire", "Wire", "vpc", "Vpc"}},
{[]string{"guests", "guest1"}, []string{"host", "Host"}},
{[]string{"guestnetworks", "1"}, []string{"guest", "Guest", "network", "Network"}},
} {
o, err := obj.Get(c.path...)
if err != nil {
t.Fatalf("get %v: %v", c.path, err)
}
for _, k := range c.keys {
if o.Contains(k) {
t.Fatalf("%v still carries join link %q", c.path, k)
}
}
}
// receiver side: parse into a fresh set, apply updates, join
mssNews := NewModelSets()
if err := obj.Unmarshal(mssNews); err != nil {
t.Fatalf("unmarshal into ModelSets: %v", err)
}
if len(mssNews.Vpcs) != 1 || len(mssNews.Wires) != 1 || len(mssNews.RouteTables) != 1 ||
len(mssNews.Networks) != 1 || len(mssNews.DnsZones) != 1 || len(mssNews.DnsRecords) != 1 ||
len(mssNews.Guests) != 1 || len(mssNews.Guestnetworks) != 1 {
t.Fatalf("round trip lost members: %s", mssNews.Stats().Dump())
}
if mssNews.Wires["wire1"] == nil || mssNews.Wires["wire1"].VpcId != "vpc1" {
t.Fatal("wire1 did not survive the round trip")
}
if mssNews.DnsRecords["rec1"] == nil || mssNews.DnsRecords["rec1"].DnsZoneId != "zone1" {
t.Fatal("rec1 did not survive the round trip")
}
local := NewModelSets()
r := local.ApplyUpdates(mssNews)
if !r.Changed {
t.Fatal("ApplyUpdates reported no change")
}
if !r.Correct {
t.Fatal("ApplyUpdates join failed")
}
if local.Wires["wire1"] == nil || local.Wires["wire1"].Vpc == nil {
t.Fatal("wire->vpc link not restored after ApplyUpdates")
}
if local.RouteTables["rt1"] == nil || local.RouteTables["rt1"].Vpc == nil {
t.Fatal("routetable->vpc link not restored after ApplyUpdates")
}
if local.DnsRecords["rec1"] == nil || local.DnsRecords["rec1"].DnsZone == nil {
t.Fatal("dnsrecord->dnszone link not restored after ApplyUpdates")
}
if local.Guestnetworks["1"] == nil || local.Guestnetworks["1"].Network == nil {
t.Fatal("guestnetwork->network link not restored after ApplyUpdates")
}
if local.Guestnetworks["1"].Guest == nil || local.Guests["guest1"].Host == nil {
t.Fatal("guestnetwork->guest / guest->host link not restored after ApplyUpdates")
}
}
+48
View File
@@ -41,6 +41,47 @@ type Worker struct {
opts *options.Options
apih *apihelper.APIHelper
// size is the guard against pushing a batch of model sets that lost most
// of its models, see modelSetsSizeGuard
size modelSetsSizeGuard
}
// modelSetsDropRatio is how much of the previous model sets has to be left for
// a run to go ahead, as a fraction of what was dropped.
const modelSetsDropRatio = 0.9
// modelSetsSizeGuard holds back a batch of model sets that lost most of its
// models. A run clears whatever the model sets do not claim (see Sweep), so a
// batch that came back incomplete would take the network configuration of the
// whole platform down with it.
//
// A drop is held back at most once, so a real shrink of the platform is only
// delayed by a round of sync rather than blocked, which also means the worker
// can never get stuck on an old batch.
type modelSetsSizeGuard struct {
baseSize int
suspended bool
}
func (g *modelSetsSizeGuard) allow(size int) bool {
if g.baseSize <= 0 {
// no baseline yet, nothing to compare against
g.baseSize = size
return true
}
if g.suspended {
// the drop was already reported once, go ahead with it
g.suspended = false
g.baseSize = size
return true
}
if float64(size) < float64(g.baseSize)*(1-modelSetsDropRatio) {
g.suspended = true
return false
}
g.baseSize = size
return true
}
func NewWorker(opts *options.Options) worker.IWorker {
@@ -116,6 +157,12 @@ func (w *Worker) run(ctx context.Context, mss *agentmodels.ModelSets) (err error
}
}()
stats := mss.Stats()
if !w.size.allow(stats.Total()) {
log.Errorf("ovn: model sets lost most of their models, skip this run: %s", stats.Dump())
return nil
}
dbUrl := w.opts.OvnNorthDatabase
if db, err := ovsutils.NormalizeDbHost(dbUrl); err != nil {
return err
@@ -123,6 +170,7 @@ func (w *Worker) run(ctx context.Context, mss *agentmodels.ModelSets) (err error
dbUrl = db
}
log.Infof("ovn: connect to ovn north database %s", dbUrl)
log.Infof("ovn: model sets stats: %s", stats.Dump())
ovnnbctl := ovnutil.NewOvnNbCtl(dbUrl)
ovndb, err := DumpOVNNorthbound(ctx, ovnnbctl)
-3
View File
@@ -262,9 +262,6 @@ show_update_cmd() {
'apigateway')
spec='apiGateway'
;;
'apimap')
spec='apiMap'
;;
'baremetal-agent')
spec='baremetalagent'
name='baremetal-agent'