dataflow-gen.go 171 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497
  1. // Package dataflow provides access to the Google Dataflow API.
  2. //
  3. // See https://cloud.google.com/dataflow
  4. //
  5. // Usage example:
  6. //
  7. // import "google.golang.org/api/dataflow/v1b3"
  8. // ...
  9. // dataflowService, err := dataflow.New(oauthHttpClient)
  10. package dataflow // import "google.golang.org/api/dataflow/v1b3"
  11. import (
  12. "bytes"
  13. "encoding/json"
  14. "errors"
  15. "fmt"
  16. context "golang.org/x/net/context"
  17. ctxhttp "golang.org/x/net/context/ctxhttp"
  18. gensupport "google.golang.org/api/gensupport"
  19. googleapi "google.golang.org/api/googleapi"
  20. "io"
  21. "net/http"
  22. "net/url"
  23. "strconv"
  24. "strings"
  25. )
  26. // Always reference these packages, just in case the auto-generated code
  27. // below doesn't.
  28. var _ = bytes.NewBuffer
  29. var _ = strconv.Itoa
  30. var _ = fmt.Sprintf
  31. var _ = json.NewDecoder
  32. var _ = io.Copy
  33. var _ = url.Parse
  34. var _ = gensupport.MarshalJSON
  35. var _ = googleapi.Version
  36. var _ = errors.New
  37. var _ = strings.Replace
  38. var _ = context.Canceled
  39. var _ = ctxhttp.Do
  40. const apiId = "dataflow:v1b3"
  41. const apiName = "dataflow"
  42. const apiVersion = "v1b3"
  43. const basePath = "https://dataflow.googleapis.com/"
  44. // OAuth2 scopes used by this API.
  45. const (
  46. // View and manage your data across Google Cloud Platform services
  47. CloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform"
  48. // View your email address
  49. UserinfoEmailScope = "https://www.googleapis.com/auth/userinfo.email"
  50. )
  51. func New(client *http.Client) (*Service, error) {
  52. if client == nil {
  53. return nil, errors.New("client is nil")
  54. }
  55. s := &Service{client: client, BasePath: basePath}
  56. s.Projects = NewProjectsService(s)
  57. return s, nil
  58. }
  59. type Service struct {
  60. client *http.Client
  61. BasePath string // API endpoint base URL
  62. UserAgent string // optional additional User-Agent fragment
  63. Projects *ProjectsService
  64. }
  65. func (s *Service) userAgent() string {
  66. if s.UserAgent == "" {
  67. return googleapi.UserAgent
  68. }
  69. return googleapi.UserAgent + " " + s.UserAgent
  70. }
  71. func NewProjectsService(s *Service) *ProjectsService {
  72. rs := &ProjectsService{s: s}
  73. rs.Jobs = NewProjectsJobsService(s)
  74. return rs
  75. }
  76. type ProjectsService struct {
  77. s *Service
  78. Jobs *ProjectsJobsService
  79. }
  80. func NewProjectsJobsService(s *Service) *ProjectsJobsService {
  81. rs := &ProjectsJobsService{s: s}
  82. rs.Messages = NewProjectsJobsMessagesService(s)
  83. rs.WorkItems = NewProjectsJobsWorkItemsService(s)
  84. return rs
  85. }
  86. type ProjectsJobsService struct {
  87. s *Service
  88. Messages *ProjectsJobsMessagesService
  89. WorkItems *ProjectsJobsWorkItemsService
  90. }
  91. func NewProjectsJobsMessagesService(s *Service) *ProjectsJobsMessagesService {
  92. rs := &ProjectsJobsMessagesService{s: s}
  93. return rs
  94. }
  95. type ProjectsJobsMessagesService struct {
  96. s *Service
  97. }
  98. func NewProjectsJobsWorkItemsService(s *Service) *ProjectsJobsWorkItemsService {
  99. rs := &ProjectsJobsWorkItemsService{s: s}
  100. return rs
  101. }
  102. type ProjectsJobsWorkItemsService struct {
  103. s *Service
  104. }
  105. // ApproximateProgress: Obsolete in favor of ApproximateReportedProgress
  106. // and ApproximateSplitRequest.
  107. type ApproximateProgress struct {
  108. // PercentComplete: Obsolete.
  109. PercentComplete float64 `json:"percentComplete,omitempty"`
  110. // Position: Obsolete.
  111. Position *Position `json:"position,omitempty"`
  112. // RemainingTime: Obsolete.
  113. RemainingTime string `json:"remainingTime,omitempty"`
  114. // ForceSendFields is a list of field names (e.g. "PercentComplete") to
  115. // unconditionally include in API requests. By default, fields with
  116. // empty values are omitted from API requests. However, any non-pointer,
  117. // non-interface field appearing in ForceSendFields will be sent to the
  118. // server regardless of whether the field is empty or not. This may be
  119. // used to include empty fields in Patch requests.
  120. ForceSendFields []string `json:"-"`
  121. }
  122. func (s *ApproximateProgress) MarshalJSON() ([]byte, error) {
  123. type noMethod ApproximateProgress
  124. raw := noMethod(*s)
  125. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  126. }
  127. // ApproximateReportedProgress: A progress measurement of a WorkItem by
  128. // a worker.
  129. type ApproximateReportedProgress struct {
  130. // ConsumedParallelism: Total amount of parallelism in the portion of
  131. // input of this work item that has already been consumed. In the first
  132. // two examples above (see remaining_parallelism), the value should be
  133. // 30 or 3 respectively. The sum of remaining_parallelism and
  134. // consumed_parallelism should equal the total amount of parallelism in
  135. // this work item. If specified, must be finite.
  136. ConsumedParallelism *ReportedParallelism `json:"consumedParallelism,omitempty"`
  137. // FractionConsumed: Completion as fraction of the input consumed, from
  138. // 0.0 (beginning, nothing consumed), to 1.0 (end of the input, entire
  139. // input consumed).
  140. FractionConsumed float64 `json:"fractionConsumed,omitempty"`
  141. // Position: A Position within the work to represent a progress.
  142. Position *Position `json:"position,omitempty"`
  143. // RemainingParallelism: Total amount of parallelism in the input of
  144. // this WorkItem that has not been consumed yet (i.e. can be delegated
  145. // to a new WorkItem via dynamic splitting). "Amount of parallelism"
  146. // refers to how many non-empty parts of the input can be read in
  147. // parallel. This does not necessarily equal number of records. An input
  148. // that can be read in parallel down to the individual records is called
  149. // "perfectly splittable". An example of non-perfectly parallelizable
  150. // input is a block-compressed file format where a block of records has
  151. // to be read as a whole, but different blocks can be read in parallel.
  152. // Examples: * If we have read 30 records out of 50 in a perfectly
  153. // splittable 50-record input, this value should be 20. * If we are
  154. // reading through block 3 in a block-compressed file consisting of 5
  155. // blocks, this value should be 2 (since blocks 4 and 5 can be processed
  156. // in parallel by new work items via dynamic splitting). * If we are
  157. // reading through the last block in a block-compressed file, or reading
  158. // or processing the last record in a perfectly splittable input, this
  159. // value should be 0, because the remainder of the work item cannot be
  160. // further split.
  161. RemainingParallelism *ReportedParallelism `json:"remainingParallelism,omitempty"`
  162. // ForceSendFields is a list of field names (e.g. "ConsumedParallelism")
  163. // to unconditionally include in API requests. By default, fields with
  164. // empty values are omitted from API requests. However, any non-pointer,
  165. // non-interface field appearing in ForceSendFields will be sent to the
  166. // server regardless of whether the field is empty or not. This may be
  167. // used to include empty fields in Patch requests.
  168. ForceSendFields []string `json:"-"`
  169. }
  170. func (s *ApproximateReportedProgress) MarshalJSON() ([]byte, error) {
  171. type noMethod ApproximateReportedProgress
  172. raw := noMethod(*s)
  173. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  174. }
  175. // ApproximateSplitRequest: A suggestion by the service to the worker to
  176. // dynamically split the WorkItem.
  177. type ApproximateSplitRequest struct {
  178. // FractionConsumed: A fraction at which to split the work item, from
  179. // 0.0 (beginning of the input) to 1.0 (end of the input).
  180. FractionConsumed float64 `json:"fractionConsumed,omitempty"`
  181. // Position: A Position at which to split the work item.
  182. Position *Position `json:"position,omitempty"`
  183. // ForceSendFields is a list of field names (e.g. "FractionConsumed") to
  184. // unconditionally include in API requests. By default, fields with
  185. // empty values are omitted from API requests. However, any non-pointer,
  186. // non-interface field appearing in ForceSendFields will be sent to the
  187. // server regardless of whether the field is empty or not. This may be
  188. // used to include empty fields in Patch requests.
  189. ForceSendFields []string `json:"-"`
  190. }
  191. func (s *ApproximateSplitRequest) MarshalJSON() ([]byte, error) {
  192. type noMethod ApproximateSplitRequest
  193. raw := noMethod(*s)
  194. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  195. }
  196. // AutoscalingSettings: Settings for WorkerPool autoscaling.
  197. type AutoscalingSettings struct {
  198. // Algorithm: The algorithm to use for autoscaling.
  199. //
  200. // Possible values:
  201. // "AUTOSCALING_ALGORITHM_UNKNOWN"
  202. // "AUTOSCALING_ALGORITHM_NONE"
  203. // "AUTOSCALING_ALGORITHM_BASIC"
  204. Algorithm string `json:"algorithm,omitempty"`
  205. // MaxNumWorkers: The maximum number of workers to cap scaling at.
  206. MaxNumWorkers int64 `json:"maxNumWorkers,omitempty"`
  207. // ForceSendFields is a list of field names (e.g. "Algorithm") to
  208. // unconditionally include in API requests. By default, fields with
  209. // empty values are omitted from API requests. However, any non-pointer,
  210. // non-interface field appearing in ForceSendFields will be sent to the
  211. // server regardless of whether the field is empty or not. This may be
  212. // used to include empty fields in Patch requests.
  213. ForceSendFields []string `json:"-"`
  214. }
  215. func (s *AutoscalingSettings) MarshalJSON() ([]byte, error) {
  216. type noMethod AutoscalingSettings
  217. raw := noMethod(*s)
  218. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  219. }
  220. // ComputationTopology: All configuration data for a particular
  221. // Computation.
  222. type ComputationTopology struct {
  223. // ComputationId: The ID of the computation.
  224. ComputationId string `json:"computationId,omitempty"`
  225. // Inputs: The inputs to the computation.
  226. Inputs []*StreamLocation `json:"inputs,omitempty"`
  227. // KeyRanges: The key ranges processed by the computation.
  228. KeyRanges []*KeyRangeLocation `json:"keyRanges,omitempty"`
  229. // Outputs: The outputs from the computation.
  230. Outputs []*StreamLocation `json:"outputs,omitempty"`
  231. // StateFamilies: The state family values.
  232. StateFamilies []*StateFamilyConfig `json:"stateFamilies,omitempty"`
  233. // SystemStageName: The system stage name.
  234. SystemStageName string `json:"systemStageName,omitempty"`
  235. // UserStageName: The user stage name.
  236. UserStageName string `json:"userStageName,omitempty"`
  237. // ForceSendFields is a list of field names (e.g. "ComputationId") to
  238. // unconditionally include in API requests. By default, fields with
  239. // empty values are omitted from API requests. However, any non-pointer,
  240. // non-interface field appearing in ForceSendFields will be sent to the
  241. // server regardless of whether the field is empty or not. This may be
  242. // used to include empty fields in Patch requests.
  243. ForceSendFields []string `json:"-"`
  244. }
  245. func (s *ComputationTopology) MarshalJSON() ([]byte, error) {
  246. type noMethod ComputationTopology
  247. raw := noMethod(*s)
  248. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  249. }
  250. // ConcatPosition: A position that encapsulates an inner position and an
  251. // index for the inner position. A ConcatPosition can be used by a
  252. // reader of a source that encapsulates a set of other sources.
  253. type ConcatPosition struct {
  254. // Index: Index of the inner source.
  255. Index int64 `json:"index,omitempty"`
  256. // Position: Position within the inner source.
  257. Position *Position `json:"position,omitempty"`
  258. // ForceSendFields is a list of field names (e.g. "Index") to
  259. // unconditionally include in API requests. By default, fields with
  260. // empty values are omitted from API requests. However, any non-pointer,
  261. // non-interface field appearing in ForceSendFields will be sent to the
  262. // server regardless of whether the field is empty or not. This may be
  263. // used to include empty fields in Patch requests.
  264. ForceSendFields []string `json:"-"`
  265. }
  266. func (s *ConcatPosition) MarshalJSON() ([]byte, error) {
  267. type noMethod ConcatPosition
  268. raw := noMethod(*s)
  269. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  270. }
  271. // CustomSourceLocation: Identifies the location of a custom souce.
  272. type CustomSourceLocation struct {
  273. // Stateful: Whether this source is stateful.
  274. Stateful bool `json:"stateful,omitempty"`
  275. // ForceSendFields is a list of field names (e.g. "Stateful") to
  276. // unconditionally include in API requests. By default, fields with
  277. // empty values are omitted from API requests. However, any non-pointer,
  278. // non-interface field appearing in ForceSendFields will be sent to the
  279. // server regardless of whether the field is empty or not. This may be
  280. // used to include empty fields in Patch requests.
  281. ForceSendFields []string `json:"-"`
  282. }
  283. func (s *CustomSourceLocation) MarshalJSON() ([]byte, error) {
  284. type noMethod CustomSourceLocation
  285. raw := noMethod(*s)
  286. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  287. }
  288. // DataDiskAssignment: Data disk assignment for a given VM instance.
  289. type DataDiskAssignment struct {
  290. // DataDisks: Mounted data disks. The order is important a data disk's
  291. // 0-based index in this list defines which persistent directory the
  292. // disk is mounted to, for example the list of {
  293. // "myproject-1014-104817-4c2-harness-0-disk-0" }, {
  294. // "myproject-1014-104817-4c2-harness-0-disk-1" }.
  295. DataDisks []string `json:"dataDisks,omitempty"`
  296. // VmInstance: VM instance name the data disks mounted to, for example
  297. // "myproject-1014-104817-4c2-harness-0".
  298. VmInstance string `json:"vmInstance,omitempty"`
  299. // ForceSendFields is a list of field names (e.g. "DataDisks") to
  300. // unconditionally include in API requests. By default, fields with
  301. // empty values are omitted from API requests. However, any non-pointer,
  302. // non-interface field appearing in ForceSendFields will be sent to the
  303. // server regardless of whether the field is empty or not. This may be
  304. // used to include empty fields in Patch requests.
  305. ForceSendFields []string `json:"-"`
  306. }
  307. func (s *DataDiskAssignment) MarshalJSON() ([]byte, error) {
  308. type noMethod DataDiskAssignment
  309. raw := noMethod(*s)
  310. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  311. }
  312. // DerivedSource: Specification of one of the bundles produced as a
  313. // result of splitting a Source (e.g. when executing a
  314. // SourceSplitRequest, or when splitting an active task using
  315. // WorkItemStatus.dynamic_source_split), relative to the source being
  316. // split.
  317. type DerivedSource struct {
  318. // DerivationMode: What source to base the produced source on (if any).
  319. //
  320. // Possible values:
  321. // "SOURCE_DERIVATION_MODE_UNKNOWN"
  322. // "SOURCE_DERIVATION_MODE_INDEPENDENT"
  323. // "SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT"
  324. // "SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT"
  325. DerivationMode string `json:"derivationMode,omitempty"`
  326. // Source: Specification of the source.
  327. Source *Source `json:"source,omitempty"`
  328. // ForceSendFields is a list of field names (e.g. "DerivationMode") to
  329. // unconditionally include in API requests. By default, fields with
  330. // empty values are omitted from API requests. However, any non-pointer,
  331. // non-interface field appearing in ForceSendFields will be sent to the
  332. // server regardless of whether the field is empty or not. This may be
  333. // used to include empty fields in Patch requests.
  334. ForceSendFields []string `json:"-"`
  335. }
  336. func (s *DerivedSource) MarshalJSON() ([]byte, error) {
  337. type noMethod DerivedSource
  338. raw := noMethod(*s)
  339. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  340. }
  341. // Disk: Describes the data disk used by a workflow job.
  342. type Disk struct {
  343. // DiskType: Disk storage type, as defined by Google Compute Engine.
  344. // This must be a disk type appropriate to the project and zone in which
  345. // the workers will run. If unknown or unspecified, the service will
  346. // attempt to choose a reasonable default. For example, the standard
  347. // persistent disk type is a resource name typically ending in
  348. // "pd-standard". If SSD persistent disks are available, the resource
  349. // name typically ends with "pd-ssd". The actual valid values are
  350. // defined the Google Compute Engine API, not by the Dataflow API;
  351. // consult the Google Compute Engine documentation for more information
  352. // about determining the set of available disk types for a particular
  353. // project and zone. Google Compute Engine Disk types are local to a
  354. // particular project in a particular zone, and so the resource name
  355. // will typically look something like this:
  356. // compute.googleapis.com/projects/
  357. // /zones//diskTypes/pd-standard
  358. DiskType string `json:"diskType,omitempty"`
  359. // MountPoint: Directory in a VM where disk is mounted.
  360. MountPoint string `json:"mountPoint,omitempty"`
  361. // SizeGb: Size of disk in GB. If zero or unspecified, the service will
  362. // attempt to choose a reasonable default.
  363. SizeGb int64 `json:"sizeGb,omitempty"`
  364. // ForceSendFields is a list of field names (e.g. "DiskType") to
  365. // unconditionally include in API requests. By default, fields with
  366. // empty values are omitted from API requests. However, any non-pointer,
  367. // non-interface field appearing in ForceSendFields will be sent to the
  368. // server regardless of whether the field is empty or not. This may be
  369. // used to include empty fields in Patch requests.
  370. ForceSendFields []string `json:"-"`
  371. }
  372. func (s *Disk) MarshalJSON() ([]byte, error) {
  373. type noMethod Disk
  374. raw := noMethod(*s)
  375. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  376. }
  377. // DynamicSourceSplit: When a task splits using
  378. // WorkItemStatus.dynamic_source_split, this message describes the two
  379. // parts of the split relative to the description of the current task's
  380. // input.
  381. type DynamicSourceSplit struct {
  382. // Primary: Primary part (continued to be processed by worker).
  383. // Specified relative to the previously-current source. Becomes current.
  384. Primary *DerivedSource `json:"primary,omitempty"`
  385. // Residual: Residual part (returned to the pool of work). Specified
  386. // relative to the previously-current source.
  387. Residual *DerivedSource `json:"residual,omitempty"`
  388. // ForceSendFields is a list of field names (e.g. "Primary") to
  389. // unconditionally include in API requests. By default, fields with
  390. // empty values are omitted from API requests. However, any non-pointer,
  391. // non-interface field appearing in ForceSendFields will be sent to the
  392. // server regardless of whether the field is empty or not. This may be
  393. // used to include empty fields in Patch requests.
  394. ForceSendFields []string `json:"-"`
  395. }
  396. func (s *DynamicSourceSplit) MarshalJSON() ([]byte, error) {
  397. type noMethod DynamicSourceSplit
  398. raw := noMethod(*s)
  399. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  400. }
  401. // Environment: Describes the environment in which a Dataflow Job runs.
  402. type Environment struct {
  403. // ClusterManagerApiService: The type of cluster manager API to use. If
  404. // unknown or unspecified, the service will attempt to choose a
  405. // reasonable default. This should be in the form of the API service
  406. // name, e.g. "compute.googleapis.com".
  407. ClusterManagerApiService string `json:"clusterManagerApiService,omitempty"`
  408. // Dataset: The dataset for the current project where various workflow
  409. // related tables are stored. The supported resource type is: Google
  410. // BigQuery: bigquery.googleapis.com/{dataset}
  411. Dataset string `json:"dataset,omitempty"`
  412. // Experiments: The list of experiments to enable.
  413. Experiments []string `json:"experiments,omitempty"`
  414. // InternalExperiments: Experimental settings.
  415. InternalExperiments EnvironmentInternalExperiments `json:"internalExperiments,omitempty"`
  416. // SdkPipelineOptions: The Dataflow SDK pipeline options specified by
  417. // the user. These options are passed through the service and are used
  418. // to recreate the SDK pipeline options on the worker in a language
  419. // agnostic and platform independent way.
  420. SdkPipelineOptions EnvironmentSdkPipelineOptions `json:"sdkPipelineOptions,omitempty"`
  421. // TempStoragePrefix: The prefix of the resources the system should use
  422. // for temporary storage. The system will append the suffix
  423. // "/temp-{JOBNAME} to this resource prefix, where {JOBNAME} is the
  424. // value of the job_name field. The resulting bucket and object prefix
  425. // is used as the prefix of the resources used to store temporary data
  426. // needed during the job execution. NOTE: This will override the value
  427. // in taskrunner_settings. The supported resource type is: Google Cloud
  428. // Storage: storage.googleapis.com/{bucket}/{object}
  429. // bucket.storage.googleapis.com/{object}
  430. TempStoragePrefix string `json:"tempStoragePrefix,omitempty"`
  431. // UserAgent: A description of the process that generated the request.
  432. UserAgent EnvironmentUserAgent `json:"userAgent,omitempty"`
  433. // Version: A structure describing which components and their versions
  434. // of the service are required in order to run the job.
  435. Version EnvironmentVersion `json:"version,omitempty"`
  436. // WorkerPools: Worker pools. At least one "harness" worker pool must be
  437. // specified in order for the job to have workers.
  438. WorkerPools []*WorkerPool `json:"workerPools,omitempty"`
  439. // ForceSendFields is a list of field names (e.g.
  440. // "ClusterManagerApiService") to unconditionally include in API
  441. // requests. By default, fields with empty values are omitted from API
  442. // requests. However, any non-pointer, non-interface field appearing in
  443. // ForceSendFields will be sent to the server regardless of whether the
  444. // field is empty or not. This may be used to include empty fields in
  445. // Patch requests.
  446. ForceSendFields []string `json:"-"`
  447. }
  448. func (s *Environment) MarshalJSON() ([]byte, error) {
  449. type noMethod Environment
  450. raw := noMethod(*s)
  451. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  452. }
  453. type EnvironmentInternalExperiments interface{}
  454. type EnvironmentSdkPipelineOptions interface{}
  455. type EnvironmentUserAgent interface{}
  456. type EnvironmentVersion interface{}
  457. // FlattenInstruction: An instruction that copies its inputs (zero or
  458. // more) to its (single) output.
  459. type FlattenInstruction struct {
  460. // Inputs: Describes the inputs to the flatten instruction.
  461. Inputs []*InstructionInput `json:"inputs,omitempty"`
  462. // ForceSendFields is a list of field names (e.g. "Inputs") to
  463. // unconditionally include in API requests. By default, fields with
  464. // empty values are omitted from API requests. However, any non-pointer,
  465. // non-interface field appearing in ForceSendFields will be sent to the
  466. // server regardless of whether the field is empty or not. This may be
  467. // used to include empty fields in Patch requests.
  468. ForceSendFields []string `json:"-"`
  469. }
  470. func (s *FlattenInstruction) MarshalJSON() ([]byte, error) {
  471. type noMethod FlattenInstruction
  472. raw := noMethod(*s)
  473. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  474. }
  475. // InstructionInput: An input of an instruction, as a reference to an
  476. // output of a producer instruction.
  477. type InstructionInput struct {
  478. // OutputNum: The output index (origin zero) within the producer.
  479. OutputNum int64 `json:"outputNum,omitempty"`
  480. // ProducerInstructionIndex: The index (origin zero) of the parallel
  481. // instruction that produces the output to be consumed by this input.
  482. // This index is relative to the list of instructions in this input's
  483. // instruction's containing MapTask.
  484. ProducerInstructionIndex int64 `json:"producerInstructionIndex,omitempty"`
  485. // ForceSendFields is a list of field names (e.g. "OutputNum") to
  486. // unconditionally include in API requests. By default, fields with
  487. // empty values are omitted from API requests. However, any non-pointer,
  488. // non-interface field appearing in ForceSendFields will be sent to the
  489. // server regardless of whether the field is empty or not. This may be
  490. // used to include empty fields in Patch requests.
  491. ForceSendFields []string `json:"-"`
  492. }
  493. func (s *InstructionInput) MarshalJSON() ([]byte, error) {
  494. type noMethod InstructionInput
  495. raw := noMethod(*s)
  496. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  497. }
  498. // InstructionOutput: An output of an instruction.
  499. type InstructionOutput struct {
  500. // Codec: The codec to use to encode data being written via this output.
  501. Codec InstructionOutputCodec `json:"codec,omitempty"`
  502. // Name: The user-provided name of this output.
  503. Name string `json:"name,omitempty"`
  504. // SystemName: System-defined name of this output. Unique across the
  505. // workflow.
  506. SystemName string `json:"systemName,omitempty"`
  507. // ForceSendFields is a list of field names (e.g. "Codec") to
  508. // unconditionally include in API requests. By default, fields with
  509. // empty values are omitted from API requests. However, any non-pointer,
  510. // non-interface field appearing in ForceSendFields will be sent to the
  511. // server regardless of whether the field is empty or not. This may be
  512. // used to include empty fields in Patch requests.
  513. ForceSendFields []string `json:"-"`
  514. }
  515. func (s *InstructionOutput) MarshalJSON() ([]byte, error) {
  516. type noMethod InstructionOutput
  517. raw := noMethod(*s)
  518. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  519. }
  520. type InstructionOutputCodec interface{}
  521. // Job: Defines a job to be run by the Dataflow service.
  522. type Job struct {
  523. // ClientRequestId: Client's unique identifier of the job, re-used by
  524. // SDK across retried attempts. If this field is set, the service will
  525. // ensure its uniqueness. That is, the request to create a job will fail
  526. // if the service has knowledge of a previously submitted job with the
  527. // same client's id and job name. The caller may, for example, use this
  528. // field to ensure idempotence of job creation across retried attempts
  529. // to create a job. By default, the field is empty and, in that case,
  530. // the service ignores it.
  531. ClientRequestId string `json:"clientRequestId,omitempty"`
  532. // CreateTime: Timestamp when job was initially created. Immutable, set
  533. // by the Dataflow service.
  534. CreateTime string `json:"createTime,omitempty"`
  535. // CurrentState: The current state of the job. Jobs are created in the
  536. // JOB_STATE_STOPPED state unless otherwise specified. A job in the
  537. // JOB_STATE_RUNNING state may asynchronously enter a terminal state.
  538. // Once a job has reached a terminal state, no further state updates may
  539. // be made. This field may be mutated by the Dataflow service; callers
  540. // cannot mutate it.
  541. //
  542. // Possible values:
  543. // "JOB_STATE_UNKNOWN"
  544. // "JOB_STATE_STOPPED"
  545. // "JOB_STATE_RUNNING"
  546. // "JOB_STATE_DONE"
  547. // "JOB_STATE_FAILED"
  548. // "JOB_STATE_CANCELLED"
  549. // "JOB_STATE_UPDATED"
  550. // "JOB_STATE_DRAINING"
  551. // "JOB_STATE_DRAINED"
  552. CurrentState string `json:"currentState,omitempty"`
  553. // CurrentStateTime: The timestamp associated with the current state.
  554. CurrentStateTime string `json:"currentStateTime,omitempty"`
  555. // Environment: Environment for the job.
  556. Environment *Environment `json:"environment,omitempty"`
  557. // ExecutionInfo: Information about how the Dataflow service will
  558. // actually run the job.
  559. ExecutionInfo *JobExecutionInfo `json:"executionInfo,omitempty"`
  560. // Id: The unique ID of this job. This field is set by the Dataflow
  561. // service when the Job is created, and is immutable for the life of the
  562. // Job.
  563. Id string `json:"id,omitempty"`
  564. // Name: The user-specified Dataflow job name. Only one Job with a given
  565. // name may exist in a project at any given time. If a caller attempts
  566. // to create a Job with the same name as an already-existing Job, the
  567. // attempt will return the existing Job. The name must match the regular
  568. // expression [a-z]([-a-z0-9]{0,38}[a-z0-9])?
  569. Name string `json:"name,omitempty"`
  570. // ProjectId: The project which owns the job.
  571. ProjectId string `json:"projectId,omitempty"`
  572. // ReplaceJobId: If this job is an update of an existing job, this field
  573. // will be the ID of the job it replaced. When sending a
  574. // CreateJobRequest, you can update a job by specifying it here. The job
  575. // named here will be stopped, and its intermediate state transferred to
  576. // this job.
  577. ReplaceJobId string `json:"replaceJobId,omitempty"`
  578. // ReplacedByJobId: If another job is an update of this job (and thus,
  579. // this job is in JOB_STATE_UPDATED), this field will contain the ID of
  580. // that job.
  581. ReplacedByJobId string `json:"replacedByJobId,omitempty"`
  582. // RequestedState: The job's requested state. UpdateJob may be used to
  583. // switch between the JOB_STATE_STOPPED and JOB_STATE_RUNNING states, by
  584. // setting requested_state. UpdateJob may also be used to directly set a
  585. // job's requested state to JOB_STATE_CANCELLED or JOB_STATE_DONE,
  586. // irrevocably terminating the job if it has not already reached a
  587. // terminal state.
  588. //
  589. // Possible values:
  590. // "JOB_STATE_UNKNOWN"
  591. // "JOB_STATE_STOPPED"
  592. // "JOB_STATE_RUNNING"
  593. // "JOB_STATE_DONE"
  594. // "JOB_STATE_FAILED"
  595. // "JOB_STATE_CANCELLED"
  596. // "JOB_STATE_UPDATED"
  597. // "JOB_STATE_DRAINING"
  598. // "JOB_STATE_DRAINED"
  599. RequestedState string `json:"requestedState,omitempty"`
  600. // Steps: The top-level steps that constitute the entire job.
  601. Steps []*Step `json:"steps,omitempty"`
  602. // TempFiles: A set of files the system should be aware of that are used
  603. // for temporary storage. These temporary files will be removed on job
  604. // completion. No duplicates are allowed. No file patterns are
  605. // supported. The supported files are: Google Cloud Storage:
  606. // storage.googleapis.com/{bucket}/{object}
  607. // bucket.storage.googleapis.com/{object}
  608. TempFiles []string `json:"tempFiles,omitempty"`
  609. // TransformNameMapping: Map of transform name prefixes of the job to be
  610. // replaced to the corresponding name prefixes of the new job.
  611. TransformNameMapping map[string]string `json:"transformNameMapping,omitempty"`
  612. // Type: The type of dataflow job.
  613. //
  614. // Possible values:
  615. // "JOB_TYPE_UNKNOWN"
  616. // "JOB_TYPE_BATCH"
  617. // "JOB_TYPE_STREAMING"
  618. Type string `json:"type,omitempty"`
  619. // ServerResponse contains the HTTP response code and headers from the
  620. // server.
  621. googleapi.ServerResponse `json:"-"`
  622. // ForceSendFields is a list of field names (e.g. "ClientRequestId") to
  623. // unconditionally include in API requests. By default, fields with
  624. // empty values are omitted from API requests. However, any non-pointer,
  625. // non-interface field appearing in ForceSendFields will be sent to the
  626. // server regardless of whether the field is empty or not. This may be
  627. // used to include empty fields in Patch requests.
  628. ForceSendFields []string `json:"-"`
  629. }
  630. func (s *Job) MarshalJSON() ([]byte, error) {
  631. type noMethod Job
  632. raw := noMethod(*s)
  633. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  634. }
  635. // JobExecutionInfo: Additional information about how a Dataflow job
  636. // will be executed which isn’t contained in the submitted job.
  637. type JobExecutionInfo struct {
  638. // Stages: A mapping from each stage to the information about that
  639. // stage.
  640. Stages map[string]JobExecutionStageInfo `json:"stages,omitempty"`
  641. // ForceSendFields is a list of field names (e.g. "Stages") to
  642. // unconditionally include in API requests. By default, fields with
  643. // empty values are omitted from API requests. However, any non-pointer,
  644. // non-interface field appearing in ForceSendFields will be sent to the
  645. // server regardless of whether the field is empty or not. This may be
  646. // used to include empty fields in Patch requests.
  647. ForceSendFields []string `json:"-"`
  648. }
  649. func (s *JobExecutionInfo) MarshalJSON() ([]byte, error) {
  650. type noMethod JobExecutionInfo
  651. raw := noMethod(*s)
  652. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  653. }
  654. // JobExecutionStageInfo: Contains information about how a particular
  655. // google.dataflow.v1beta3.Step will be executed.
  656. type JobExecutionStageInfo struct {
  657. // StepName: The steps associated with the execution stage. Note that
  658. // stages may have several steps, and that a given step might be run by
  659. // more than one stage.
  660. StepName []string `json:"stepName,omitempty"`
  661. // ForceSendFields is a list of field names (e.g. "StepName") to
  662. // unconditionally include in API requests. By default, fields with
  663. // empty values are omitted from API requests. However, any non-pointer,
  664. // non-interface field appearing in ForceSendFields will be sent to the
  665. // server regardless of whether the field is empty or not. This may be
  666. // used to include empty fields in Patch requests.
  667. ForceSendFields []string `json:"-"`
  668. }
  669. func (s *JobExecutionStageInfo) MarshalJSON() ([]byte, error) {
  670. type noMethod JobExecutionStageInfo
  671. raw := noMethod(*s)
  672. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  673. }
  674. // JobMessage: A particular message pertaining to a Dataflow job.
  675. type JobMessage struct {
  676. // Id: Identifies the message. This is automatically generated by the
  677. // service; the caller should treat it as an opaque string.
  678. Id string `json:"id,omitempty"`
  679. // MessageImportance: Importance level of the message.
  680. //
  681. // Possible values:
  682. // "JOB_MESSAGE_IMPORTANCE_UNKNOWN"
  683. // "JOB_MESSAGE_DEBUG"
  684. // "JOB_MESSAGE_DETAILED"
  685. // "JOB_MESSAGE_BASIC"
  686. // "JOB_MESSAGE_WARNING"
  687. // "JOB_MESSAGE_ERROR"
  688. MessageImportance string `json:"messageImportance,omitempty"`
  689. // MessageText: The text of the message.
  690. MessageText string `json:"messageText,omitempty"`
  691. // Time: The timestamp of the message.
  692. Time string `json:"time,omitempty"`
  693. // ForceSendFields is a list of field names (e.g. "Id") to
  694. // unconditionally include in API requests. By default, fields with
  695. // empty values are omitted from API requests. However, any non-pointer,
  696. // non-interface field appearing in ForceSendFields will be sent to the
  697. // server regardless of whether the field is empty or not. This may be
  698. // used to include empty fields in Patch requests.
  699. ForceSendFields []string `json:"-"`
  700. }
  701. func (s *JobMessage) MarshalJSON() ([]byte, error) {
  702. type noMethod JobMessage
  703. raw := noMethod(*s)
  704. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  705. }
  706. // JobMetrics: JobMetrics contains a collection of metrics descibing the
  707. // detailed progress of a Dataflow job. Metrics correspond to
  708. // user-defined and system-defined metrics in the job. This resource
  709. // captures only the most recent values of each metric; time-series data
  710. // can be queried for them (under the same metric names) from Cloud
  711. // Monitoring.
  712. type JobMetrics struct {
  713. // MetricTime: Timestamp as of which metric values are current.
  714. MetricTime string `json:"metricTime,omitempty"`
  715. // Metrics: All metrics for this job.
  716. Metrics []*MetricUpdate `json:"metrics,omitempty"`
  717. // ServerResponse contains the HTTP response code and headers from the
  718. // server.
  719. googleapi.ServerResponse `json:"-"`
  720. // ForceSendFields is a list of field names (e.g. "MetricTime") to
  721. // unconditionally include in API requests. By default, fields with
  722. // empty values are omitted from API requests. However, any non-pointer,
  723. // non-interface field appearing in ForceSendFields will be sent to the
  724. // server regardless of whether the field is empty or not. This may be
  725. // used to include empty fields in Patch requests.
  726. ForceSendFields []string `json:"-"`
  727. }
  728. func (s *JobMetrics) MarshalJSON() ([]byte, error) {
  729. type noMethod JobMetrics
  730. raw := noMethod(*s)
  731. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  732. }
  733. // KeyRangeDataDiskAssignment: Data disk assignment information for a
  734. // specific key-range of a sharded computation. Currently we only
  735. // support UTF-8 character splits to simplify encoding into JSON.
  736. type KeyRangeDataDiskAssignment struct {
  737. // DataDisk: The name of the data disk where data for this range is
  738. // stored. This name is local to the Google Cloud Platform project and
  739. // uniquely identifies the disk within that project, for example
  740. // "myproject-1014-104817-4c2-harness-0-disk-1".
  741. DataDisk string `json:"dataDisk,omitempty"`
  742. // End: The end (exclusive) of the key range.
  743. End string `json:"end,omitempty"`
  744. // Start: The start (inclusive) of the key range.
  745. Start string `json:"start,omitempty"`
  746. // ForceSendFields is a list of field names (e.g. "DataDisk") to
  747. // unconditionally include in API requests. By default, fields with
  748. // empty values are omitted from API requests. However, any non-pointer,
  749. // non-interface field appearing in ForceSendFields will be sent to the
  750. // server regardless of whether the field is empty or not. This may be
  751. // used to include empty fields in Patch requests.
  752. ForceSendFields []string `json:"-"`
  753. }
  754. func (s *KeyRangeDataDiskAssignment) MarshalJSON() ([]byte, error) {
  755. type noMethod KeyRangeDataDiskAssignment
  756. raw := noMethod(*s)
  757. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  758. }
  759. // KeyRangeLocation: Location information for a specific key-range of a
  760. // sharded computation. Currently we only support UTF-8 character splits
  761. // to simplify encoding into JSON.
  762. type KeyRangeLocation struct {
  763. // DataDisk: The name of the data disk where data for this range is
  764. // stored. This name is local to the Google Cloud Platform project and
  765. // uniquely identifies the disk within that project, for example
  766. // "myproject-1014-104817-4c2-harness-0-disk-1".
  767. DataDisk string `json:"dataDisk,omitempty"`
  768. // DeliveryEndpoint: The physical location of this range assignment to
  769. // be used for streaming computation cross-worker message delivery.
  770. DeliveryEndpoint string `json:"deliveryEndpoint,omitempty"`
  771. // End: The end (exclusive) of the key range.
  772. End string `json:"end,omitempty"`
  773. // PersistentDirectory: The location of the persistent state for this
  774. // range, as a persistent directory in the worker local filesystem.
  775. PersistentDirectory string `json:"persistentDirectory,omitempty"`
  776. // Start: The start (inclusive) of the key range.
  777. Start string `json:"start,omitempty"`
  778. // ForceSendFields is a list of field names (e.g. "DataDisk") to
  779. // unconditionally include in API requests. By default, fields with
  780. // empty values are omitted from API requests. However, any non-pointer,
  781. // non-interface field appearing in ForceSendFields will be sent to the
  782. // server regardless of whether the field is empty or not. This may be
  783. // used to include empty fields in Patch requests.
  784. ForceSendFields []string `json:"-"`
  785. }
  786. func (s *KeyRangeLocation) MarshalJSON() ([]byte, error) {
  787. type noMethod KeyRangeLocation
  788. raw := noMethod(*s)
  789. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  790. }
  791. // LeaseWorkItemRequest: Request to lease WorkItems.
  792. type LeaseWorkItemRequest struct {
  793. // CurrentWorkerTime: The current timestamp at the worker.
  794. CurrentWorkerTime string `json:"currentWorkerTime,omitempty"`
  795. // RequestedLeaseDuration: The initial lease period.
  796. RequestedLeaseDuration string `json:"requestedLeaseDuration,omitempty"`
  797. // WorkItemTypes: Filter for WorkItem type.
  798. WorkItemTypes []string `json:"workItemTypes,omitempty"`
  799. // WorkerCapabilities: Worker capabilities. WorkItems might be limited
  800. // to workers with specific capabilities.
  801. WorkerCapabilities []string `json:"workerCapabilities,omitempty"`
  802. // WorkerId: Identifies the worker leasing work -- typically the ID of
  803. // the virtual machine running the worker.
  804. WorkerId string `json:"workerId,omitempty"`
  805. // ForceSendFields is a list of field names (e.g. "CurrentWorkerTime")
  806. // to unconditionally include in API requests. By default, fields with
  807. // empty values are omitted from API requests. However, any non-pointer,
  808. // non-interface field appearing in ForceSendFields will be sent to the
  809. // server regardless of whether the field is empty or not. This may be
  810. // used to include empty fields in Patch requests.
  811. ForceSendFields []string `json:"-"`
  812. }
  813. func (s *LeaseWorkItemRequest) MarshalJSON() ([]byte, error) {
  814. type noMethod LeaseWorkItemRequest
  815. raw := noMethod(*s)
  816. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  817. }
  818. // LeaseWorkItemResponse: Response to a request to lease WorkItems.
  819. type LeaseWorkItemResponse struct {
  820. // WorkItems: A list of the leased WorkItems.
  821. WorkItems []*WorkItem `json:"workItems,omitempty"`
  822. // ServerResponse contains the HTTP response code and headers from the
  823. // server.
  824. googleapi.ServerResponse `json:"-"`
  825. // ForceSendFields is a list of field names (e.g. "WorkItems") to
  826. // unconditionally include in API requests. By default, fields with
  827. // empty values are omitted from API requests. However, any non-pointer,
  828. // non-interface field appearing in ForceSendFields will be sent to the
  829. // server regardless of whether the field is empty or not. This may be
  830. // used to include empty fields in Patch requests.
  831. ForceSendFields []string `json:"-"`
  832. }
  833. func (s *LeaseWorkItemResponse) MarshalJSON() ([]byte, error) {
  834. type noMethod LeaseWorkItemResponse
  835. raw := noMethod(*s)
  836. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  837. }
  838. // ListJobMessagesResponse: Response to a request to list job messages.
  839. type ListJobMessagesResponse struct {
  840. // JobMessages: Messages in ascending timestamp order.
  841. JobMessages []*JobMessage `json:"jobMessages,omitempty"`
  842. // NextPageToken: The token to obtain the next page of results if there
  843. // are more.
  844. NextPageToken string `json:"nextPageToken,omitempty"`
  845. // ServerResponse contains the HTTP response code and headers from the
  846. // server.
  847. googleapi.ServerResponse `json:"-"`
  848. // ForceSendFields is a list of field names (e.g. "JobMessages") to
  849. // unconditionally include in API requests. By default, fields with
  850. // empty values are omitted from API requests. However, any non-pointer,
  851. // non-interface field appearing in ForceSendFields will be sent to the
  852. // server regardless of whether the field is empty or not. This may be
  853. // used to include empty fields in Patch requests.
  854. ForceSendFields []string `json:"-"`
  855. }
  856. func (s *ListJobMessagesResponse) MarshalJSON() ([]byte, error) {
  857. type noMethod ListJobMessagesResponse
  858. raw := noMethod(*s)
  859. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  860. }
  861. // ListJobsResponse: Response to a request to list Dataflow jobs. This
  862. // may be a partial response, depending on the page size in the
  863. // ListJobsRequest.
  864. type ListJobsResponse struct {
  865. // Jobs: A subset of the requested job information.
  866. Jobs []*Job `json:"jobs,omitempty"`
  867. // NextPageToken: Set if there may be more results than fit in this
  868. // response.
  869. NextPageToken string `json:"nextPageToken,omitempty"`
  870. // ServerResponse contains the HTTP response code and headers from the
  871. // server.
  872. googleapi.ServerResponse `json:"-"`
  873. // ForceSendFields is a list of field names (e.g. "Jobs") to
  874. // unconditionally include in API requests. By default, fields with
  875. // empty values are omitted from API requests. However, any non-pointer,
  876. // non-interface field appearing in ForceSendFields will be sent to the
  877. // server regardless of whether the field is empty or not. This may be
  878. // used to include empty fields in Patch requests.
  879. ForceSendFields []string `json:"-"`
  880. }
  881. func (s *ListJobsResponse) MarshalJSON() ([]byte, error) {
  882. type noMethod ListJobsResponse
  883. raw := noMethod(*s)
  884. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  885. }
  886. // MapTask: MapTask consists of an ordered set of instructions, each of
  887. // which describes one particular low-level operation for the worker to
  888. // perform in order to accomplish the MapTask's WorkItem. Each
  889. // instruction must appear in the list before any instructions which
  890. // depends on its output.
  891. type MapTask struct {
  892. // Instructions: The instructions in the MapTask.
  893. Instructions []*ParallelInstruction `json:"instructions,omitempty"`
  894. // StageName: System-defined name of the stage containing this MapTask.
  895. // Unique across the workflow.
  896. StageName string `json:"stageName,omitempty"`
  897. // SystemName: System-defined name of this MapTask. Unique across the
  898. // workflow.
  899. SystemName string `json:"systemName,omitempty"`
  900. // ForceSendFields is a list of field names (e.g. "Instructions") to
  901. // unconditionally include in API requests. By default, fields with
  902. // empty values are omitted from API requests. However, any non-pointer,
  903. // non-interface field appearing in ForceSendFields will be sent to the
  904. // server regardless of whether the field is empty or not. This may be
  905. // used to include empty fields in Patch requests.
  906. ForceSendFields []string `json:"-"`
  907. }
  908. func (s *MapTask) MarshalJSON() ([]byte, error) {
  909. type noMethod MapTask
  910. raw := noMethod(*s)
  911. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  912. }
  913. // MetricStructuredName: Identifies a metric, by describing the source
  914. // which generated the metric.
  915. type MetricStructuredName struct {
  916. // Context: Zero or more labeled fields which identify the part of the
  917. // job this metric is associated with, such as the name of a step or
  918. // collection. For example, built-in counters associated with steps will
  919. // have context['step'] = . Counters associated with PCollections in the
  920. // SDK will have context['pcollection'] =
  921. // .
  922. Context map[string]string `json:"context,omitempty"`
  923. // Name: Worker-defined metric name.
  924. Name string `json:"name,omitempty"`
  925. // Origin: Origin (namespace) of metric name. May be blank for
  926. // user-define metrics; will be "dataflow" for metrics defined by the
  927. // Dataflow service or SDK.
  928. Origin string `json:"origin,omitempty"`
  929. // ForceSendFields is a list of field names (e.g. "Context") to
  930. // unconditionally include in API requests. By default, fields with
  931. // empty values are omitted from API requests. However, any non-pointer,
  932. // non-interface field appearing in ForceSendFields will be sent to the
  933. // server regardless of whether the field is empty or not. This may be
  934. // used to include empty fields in Patch requests.
  935. ForceSendFields []string `json:"-"`
  936. }
  937. func (s *MetricStructuredName) MarshalJSON() ([]byte, error) {
  938. type noMethod MetricStructuredName
  939. raw := noMethod(*s)
  940. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  941. }
  942. // MetricUpdate: Describes the state of a metric.
  943. type MetricUpdate struct {
  944. // Cumulative: True if this metric is reported as the total cumulative
  945. // aggregate value accumulated since the worker started working on this
  946. // WorkItem. By default this is false, indicating that this metric is
  947. // reported as a delta that is not associated with any WorkItem.
  948. Cumulative bool `json:"cumulative,omitempty"`
  949. // Internal: Worker-computed aggregate value for internal use by the
  950. // Dataflow service.
  951. Internal interface{} `json:"internal,omitempty"`
  952. // Kind: Metric aggregation kind. The possible metric aggregation kinds
  953. // are "Sum", "Max", "Min", "Mean", "Set", "And", and "Or". The
  954. // specified aggregation kind is case-insensitive. If omitted, this is
  955. // not an aggregated value but instead a single metric sample value.
  956. Kind string `json:"kind,omitempty"`
  957. // MeanCount: Worker-computed aggregate value for the "Mean" aggregation
  958. // kind. This holds the count of the aggregated values and is used in
  959. // combination with mean_sum above to obtain the actual mean aggregate
  960. // value. The only possible value type is Long.
  961. MeanCount interface{} `json:"meanCount,omitempty"`
  962. // MeanSum: Worker-computed aggregate value for the "Mean" aggregation
  963. // kind. This holds the sum of the aggregated values and is used in
  964. // combination with mean_count below to obtain the actual mean aggregate
  965. // value. The only possible value types are Long and Double.
  966. MeanSum interface{} `json:"meanSum,omitempty"`
  967. // Name: Name of the metric.
  968. Name *MetricStructuredName `json:"name,omitempty"`
  969. // Scalar: Worker-computed aggregate value for aggregation kinds "Sum",
  970. // "Max", "Min", "And", and "Or". The possible value types are Long,
  971. // Double, and Boolean.
  972. Scalar interface{} `json:"scalar,omitempty"`
  973. // Set: Worker-computed aggregate value for the "Set" aggregation kind.
  974. // The only possible value type is a list of Values whose type can be
  975. // Long, Double, or String, according to the metric's type. All Values
  976. // in the list must be of the same type.
  977. Set interface{} `json:"set,omitempty"`
  978. // UpdateTime: Timestamp associated with the metric value. Optional when
  979. // workers are reporting work progress; it will be filled in responses
  980. // from the metrics API.
  981. UpdateTime string `json:"updateTime,omitempty"`
  982. // ForceSendFields is a list of field names (e.g. "Cumulative") to
  983. // unconditionally include in API requests. By default, fields with
  984. // empty values are omitted from API requests. However, any non-pointer,
  985. // non-interface field appearing in ForceSendFields will be sent to the
  986. // server regardless of whether the field is empty or not. This may be
  987. // used to include empty fields in Patch requests.
  988. ForceSendFields []string `json:"-"`
  989. }
  990. func (s *MetricUpdate) MarshalJSON() ([]byte, error) {
  991. type noMethod MetricUpdate
  992. raw := noMethod(*s)
  993. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  994. }
  995. // MountedDataDisk: Describes mounted data disk.
  996. type MountedDataDisk struct {
  997. // DataDisk: The name of the data disk. This name is local to the Google
  998. // Cloud Platform project and uniquely identifies the disk within that
  999. // project, for example "myproject-1014-104817-4c2-harness-0-disk-1".
  1000. DataDisk string `json:"dataDisk,omitempty"`
  1001. // ForceSendFields is a list of field names (e.g. "DataDisk") to
  1002. // unconditionally include in API requests. By default, fields with
  1003. // empty values are omitted from API requests. However, any non-pointer,
  1004. // non-interface field appearing in ForceSendFields will be sent to the
  1005. // server regardless of whether the field is empty or not. This may be
  1006. // used to include empty fields in Patch requests.
  1007. ForceSendFields []string `json:"-"`
  1008. }
  1009. func (s *MountedDataDisk) MarshalJSON() ([]byte, error) {
  1010. type noMethod MountedDataDisk
  1011. raw := noMethod(*s)
  1012. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1013. }
  1014. // MultiOutputInfo: Information about an output of a multi-output DoFn.
  1015. type MultiOutputInfo struct {
  1016. // Tag: The id of the tag the user code will emit to this output by;
  1017. // this should correspond to the tag of some SideInputInfo.
  1018. Tag string `json:"tag,omitempty"`
  1019. // ForceSendFields is a list of field names (e.g. "Tag") to
  1020. // unconditionally include in API requests. By default, fields with
  1021. // empty values are omitted from API requests. However, any non-pointer,
  1022. // non-interface field appearing in ForceSendFields will be sent to the
  1023. // server regardless of whether the field is empty or not. This may be
  1024. // used to include empty fields in Patch requests.
  1025. ForceSendFields []string `json:"-"`
  1026. }
  1027. func (s *MultiOutputInfo) MarshalJSON() ([]byte, error) {
  1028. type noMethod MultiOutputInfo
  1029. raw := noMethod(*s)
  1030. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1031. }
  1032. // Package: Packages that need to be installed in order for a worker to
  1033. // run the steps of the Dataflow job which will be assigned to its
  1034. // worker pool. This is the mechanism by which the SDK causes code to be
  1035. // loaded onto the workers. For example, the Dataflow Java SDK might use
  1036. // this to install jars containing the user's code and all of the
  1037. // various dependencies (libraries, data files, etc) required in order
  1038. // for that code to run.
  1039. type Package struct {
  1040. // Location: The resource to read the package from. The supported
  1041. // resource type is: Google Cloud Storage:
  1042. // storage.googleapis.com/{bucket} bucket.storage.googleapis.com/
  1043. Location string `json:"location,omitempty"`
  1044. // Name: The name of the package.
  1045. Name string `json:"name,omitempty"`
  1046. // ForceSendFields is a list of field names (e.g. "Location") to
  1047. // unconditionally include in API requests. By default, fields with
  1048. // empty values are omitted from API requests. However, any non-pointer,
  1049. // non-interface field appearing in ForceSendFields will be sent to the
  1050. // server regardless of whether the field is empty or not. This may be
  1051. // used to include empty fields in Patch requests.
  1052. ForceSendFields []string `json:"-"`
  1053. }
  1054. func (s *Package) MarshalJSON() ([]byte, error) {
  1055. type noMethod Package
  1056. raw := noMethod(*s)
  1057. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1058. }
  1059. // ParDoInstruction: An instruction that does a ParDo operation. Takes
  1060. // one main input and zero or more side inputs, and produces zero or
  1061. // more outputs. Runs user code.
  1062. type ParDoInstruction struct {
  1063. // Input: The input.
  1064. Input *InstructionInput `json:"input,omitempty"`
  1065. // MultiOutputInfos: Information about each of the outputs, if user_fn
  1066. // is a MultiDoFn.
  1067. MultiOutputInfos []*MultiOutputInfo `json:"multiOutputInfos,omitempty"`
  1068. // NumOutputs: The number of outputs.
  1069. NumOutputs int64 `json:"numOutputs,omitempty"`
  1070. // SideInputs: Zero or more side inputs.
  1071. SideInputs []*SideInputInfo `json:"sideInputs,omitempty"`
  1072. // UserFn: The user function to invoke.
  1073. UserFn ParDoInstructionUserFn `json:"userFn,omitempty"`
  1074. // ForceSendFields is a list of field names (e.g. "Input") to
  1075. // unconditionally include in API requests. By default, fields with
  1076. // empty values are omitted from API requests. However, any non-pointer,
  1077. // non-interface field appearing in ForceSendFields will be sent to the
  1078. // server regardless of whether the field is empty or not. This may be
  1079. // used to include empty fields in Patch requests.
  1080. ForceSendFields []string `json:"-"`
  1081. }
  1082. func (s *ParDoInstruction) MarshalJSON() ([]byte, error) {
  1083. type noMethod ParDoInstruction
  1084. raw := noMethod(*s)
  1085. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1086. }
  1087. type ParDoInstructionUserFn interface{}
  1088. // ParallelInstruction: Describes a particular operation comprising a
  1089. // MapTask.
  1090. type ParallelInstruction struct {
  1091. // Flatten: Additional information for Flatten instructions.
  1092. Flatten *FlattenInstruction `json:"flatten,omitempty"`
  1093. // Name: User-provided name of this operation.
  1094. Name string `json:"name,omitempty"`
  1095. // Outputs: Describes the outputs of the instruction.
  1096. Outputs []*InstructionOutput `json:"outputs,omitempty"`
  1097. // ParDo: Additional information for ParDo instructions.
  1098. ParDo *ParDoInstruction `json:"parDo,omitempty"`
  1099. // PartialGroupByKey: Additional information for PartialGroupByKey
  1100. // instructions.
  1101. PartialGroupByKey *PartialGroupByKeyInstruction `json:"partialGroupByKey,omitempty"`
  1102. // Read: Additional information for Read instructions.
  1103. Read *ReadInstruction `json:"read,omitempty"`
  1104. // SystemName: System-defined name of this operation. Unique across the
  1105. // workflow.
  1106. SystemName string `json:"systemName,omitempty"`
  1107. // Write: Additional information for Write instructions.
  1108. Write *WriteInstruction `json:"write,omitempty"`
  1109. // ForceSendFields is a list of field names (e.g. "Flatten") to
  1110. // unconditionally include in API requests. By default, fields with
  1111. // empty values are omitted from API requests. However, any non-pointer,
  1112. // non-interface field appearing in ForceSendFields will be sent to the
  1113. // server regardless of whether the field is empty or not. This may be
  1114. // used to include empty fields in Patch requests.
  1115. ForceSendFields []string `json:"-"`
  1116. }
  1117. func (s *ParallelInstruction) MarshalJSON() ([]byte, error) {
  1118. type noMethod ParallelInstruction
  1119. raw := noMethod(*s)
  1120. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1121. }
  1122. // PartialGroupByKeyInstruction: An instruction that does a partial
  1123. // group-by-key. One input and one output.
  1124. type PartialGroupByKeyInstruction struct {
  1125. // Input: Describes the input to the partial group-by-key instruction.
  1126. Input *InstructionInput `json:"input,omitempty"`
  1127. // InputElementCodec: The codec to use for interpreting an element in
  1128. // the input PTable.
  1129. InputElementCodec PartialGroupByKeyInstructionInputElementCodec `json:"inputElementCodec,omitempty"`
  1130. // SideInputs: Zero or more side inputs.
  1131. SideInputs []*SideInputInfo `json:"sideInputs,omitempty"`
  1132. // ValueCombiningFn: The value combining function to invoke.
  1133. ValueCombiningFn PartialGroupByKeyInstructionValueCombiningFn `json:"valueCombiningFn,omitempty"`
  1134. // ForceSendFields is a list of field names (e.g. "Input") to
  1135. // unconditionally include in API requests. By default, fields with
  1136. // empty values are omitted from API requests. However, any non-pointer,
  1137. // non-interface field appearing in ForceSendFields will be sent to the
  1138. // server regardless of whether the field is empty or not. This may be
  1139. // used to include empty fields in Patch requests.
  1140. ForceSendFields []string `json:"-"`
  1141. }
  1142. func (s *PartialGroupByKeyInstruction) MarshalJSON() ([]byte, error) {
  1143. type noMethod PartialGroupByKeyInstruction
  1144. raw := noMethod(*s)
  1145. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1146. }
  1147. type PartialGroupByKeyInstructionInputElementCodec interface{}
  1148. type PartialGroupByKeyInstructionValueCombiningFn interface{}
  1149. // Position: Position defines a position within a collection of data.
  1150. // The value can be either the end position, a key (used with ordered
  1151. // collections), a byte offset, or a record index.
  1152. type Position struct {
  1153. // ByteOffset: Position is a byte offset.
  1154. ByteOffset int64 `json:"byteOffset,omitempty,string"`
  1155. // ConcatPosition: CloudPosition is a concat position.
  1156. ConcatPosition *ConcatPosition `json:"concatPosition,omitempty"`
  1157. // End: Position is past all other positions. Also useful for the end
  1158. // position of an unbounded range.
  1159. End bool `json:"end,omitempty"`
  1160. // Key: Position is a string key, ordered lexicographically.
  1161. Key string `json:"key,omitempty"`
  1162. // RecordIndex: Position is a record index.
  1163. RecordIndex int64 `json:"recordIndex,omitempty,string"`
  1164. // ShufflePosition: CloudPosition is a base64 encoded
  1165. // BatchShufflePosition (with FIXED sharding).
  1166. ShufflePosition string `json:"shufflePosition,omitempty"`
  1167. // ForceSendFields is a list of field names (e.g. "ByteOffset") to
  1168. // unconditionally include in API requests. By default, fields with
  1169. // empty values are omitted from API requests. However, any non-pointer,
  1170. // non-interface field appearing in ForceSendFields will be sent to the
  1171. // server regardless of whether the field is empty or not. This may be
  1172. // used to include empty fields in Patch requests.
  1173. ForceSendFields []string `json:"-"`
  1174. }
  1175. func (s *Position) MarshalJSON() ([]byte, error) {
  1176. type noMethod Position
  1177. raw := noMethod(*s)
  1178. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1179. }
  1180. // PubsubLocation: Identifies a pubsub location to use for transferring
  1181. // data into or out of a streaming Dataflow job.
  1182. type PubsubLocation struct {
  1183. // DropLateData: Indicates whether the pipeline allows late-arriving
  1184. // data.
  1185. DropLateData bool `json:"dropLateData,omitempty"`
  1186. // IdLabel: If set, contains a pubsub label from which to extract record
  1187. // ids. If left empty, record deduplication will be strictly best
  1188. // effort.
  1189. IdLabel string `json:"idLabel,omitempty"`
  1190. // Subscription: A pubsub subscription, in the form of
  1191. // "pubsub.googleapis.com/subscriptions/
  1192. // /"
  1193. Subscription string `json:"subscription,omitempty"`
  1194. // TimestampLabel: If set, contains a pubsub label from which to extract
  1195. // record timestamps. If left empty, record timestamps will be generated
  1196. // upon arrival.
  1197. TimestampLabel string `json:"timestampLabel,omitempty"`
  1198. // Topic: A pubsub topic, in the form of
  1199. // "pubsub.googleapis.com/topics/
  1200. // /"
  1201. Topic string `json:"topic,omitempty"`
  1202. // TrackingSubscription: If set, specifies the pubsub subscription that
  1203. // will be used for tracking custom time timestamps for watermark
  1204. // estimation.
  1205. TrackingSubscription string `json:"trackingSubscription,omitempty"`
  1206. // ForceSendFields is a list of field names (e.g. "DropLateData") to
  1207. // unconditionally include in API requests. By default, fields with
  1208. // empty values are omitted from API requests. However, any non-pointer,
  1209. // non-interface field appearing in ForceSendFields will be sent to the
  1210. // server regardless of whether the field is empty or not. This may be
  1211. // used to include empty fields in Patch requests.
  1212. ForceSendFields []string `json:"-"`
  1213. }
  1214. func (s *PubsubLocation) MarshalJSON() ([]byte, error) {
  1215. type noMethod PubsubLocation
  1216. raw := noMethod(*s)
  1217. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1218. }
  1219. // ReadInstruction: An instruction that reads records. Takes no inputs,
  1220. // produces one output.
  1221. type ReadInstruction struct {
  1222. // Source: The source to read from.
  1223. Source *Source `json:"source,omitempty"`
  1224. // ForceSendFields is a list of field names (e.g. "Source") to
  1225. // unconditionally include in API requests. By default, fields with
  1226. // empty values are omitted from API requests. However, any non-pointer,
  1227. // non-interface field appearing in ForceSendFields will be sent to the
  1228. // server regardless of whether the field is empty or not. This may be
  1229. // used to include empty fields in Patch requests.
  1230. ForceSendFields []string `json:"-"`
  1231. }
  1232. func (s *ReadInstruction) MarshalJSON() ([]byte, error) {
  1233. type noMethod ReadInstruction
  1234. raw := noMethod(*s)
  1235. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1236. }
  1237. // ReportWorkItemStatusRequest: Request to report the status of
  1238. // WorkItems.
  1239. type ReportWorkItemStatusRequest struct {
  1240. // CurrentWorkerTime: The current timestamp at the worker.
  1241. CurrentWorkerTime string `json:"currentWorkerTime,omitempty"`
  1242. // WorkItemStatuses: The order is unimportant, except that the order of
  1243. // the WorkItemServiceState messages in the ReportWorkItemStatusResponse
  1244. // corresponds to the order of WorkItemStatus messages here.
  1245. WorkItemStatuses []*WorkItemStatus `json:"workItemStatuses,omitempty"`
  1246. // WorkerId: The ID of the worker reporting the WorkItem status. If this
  1247. // does not match the ID of the worker which the Dataflow service
  1248. // believes currently has the lease on the WorkItem, the report will be
  1249. // dropped (with an error response).
  1250. WorkerId string `json:"workerId,omitempty"`
  1251. // ForceSendFields is a list of field names (e.g. "CurrentWorkerTime")
  1252. // to unconditionally include in API requests. By default, fields with
  1253. // empty values are omitted from API requests. However, any non-pointer,
  1254. // non-interface field appearing in ForceSendFields will be sent to the
  1255. // server regardless of whether the field is empty or not. This may be
  1256. // used to include empty fields in Patch requests.
  1257. ForceSendFields []string `json:"-"`
  1258. }
  1259. func (s *ReportWorkItemStatusRequest) MarshalJSON() ([]byte, error) {
  1260. type noMethod ReportWorkItemStatusRequest
  1261. raw := noMethod(*s)
  1262. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1263. }
  1264. // ReportWorkItemStatusResponse: Response from a request to report the
  1265. // status of WorkItems.
  1266. type ReportWorkItemStatusResponse struct {
  1267. // WorkItemServiceStates: A set of messages indicating the service-side
  1268. // state for each WorkItem whose status was reported, in the same order
  1269. // as the WorkItemStatus messages in the ReportWorkItemStatusRequest
  1270. // which resulting in this response.
  1271. WorkItemServiceStates []*WorkItemServiceState `json:"workItemServiceStates,omitempty"`
  1272. // ServerResponse contains the HTTP response code and headers from the
  1273. // server.
  1274. googleapi.ServerResponse `json:"-"`
  1275. // ForceSendFields is a list of field names (e.g.
  1276. // "WorkItemServiceStates") to unconditionally include in API requests.
  1277. // By default, fields with empty values are omitted from API requests.
  1278. // However, any non-pointer, non-interface field appearing in
  1279. // ForceSendFields will be sent to the server regardless of whether the
  1280. // field is empty or not. This may be used to include empty fields in
  1281. // Patch requests.
  1282. ForceSendFields []string `json:"-"`
  1283. }
  1284. func (s *ReportWorkItemStatusResponse) MarshalJSON() ([]byte, error) {
  1285. type noMethod ReportWorkItemStatusResponse
  1286. raw := noMethod(*s)
  1287. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1288. }
  1289. // ReportedParallelism: Represents the level of parallelism in a
  1290. // WorkItem's input, reported by the worker.
  1291. type ReportedParallelism struct {
  1292. // IsInfinite: Specifies whether the parallelism is infinite. If true,
  1293. // "value" is ignored. Infinite parallelism means the service will
  1294. // assume that the work item can always be split into more non-empty
  1295. // work items by dynamic splitting. This is a work-around for lack of
  1296. // support for infinity by the current JSON-based Java RPC stack.
  1297. IsInfinite bool `json:"isInfinite,omitempty"`
  1298. // Value: Specifies the level of parallelism in case it is finite.
  1299. Value float64 `json:"value,omitempty"`
  1300. // ForceSendFields is a list of field names (e.g. "IsInfinite") to
  1301. // unconditionally include in API requests. By default, fields with
  1302. // empty values are omitted from API requests. However, any non-pointer,
  1303. // non-interface field appearing in ForceSendFields will be sent to the
  1304. // server regardless of whether the field is empty or not. This may be
  1305. // used to include empty fields in Patch requests.
  1306. ForceSendFields []string `json:"-"`
  1307. }
  1308. func (s *ReportedParallelism) MarshalJSON() ([]byte, error) {
  1309. type noMethod ReportedParallelism
  1310. raw := noMethod(*s)
  1311. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1312. }
  1313. // SendWorkerMessagesRequest: A request for sending worker messages to
  1314. // the service.
  1315. type SendWorkerMessagesRequest struct {
  1316. // WorkerMessages: The WorkerMessages to send.
  1317. WorkerMessages []*WorkerMessage `json:"workerMessages,omitempty"`
  1318. // ForceSendFields is a list of field names (e.g. "WorkerMessages") to
  1319. // unconditionally include in API requests. By default, fields with
  1320. // empty values are omitted from API requests. However, any non-pointer,
  1321. // non-interface field appearing in ForceSendFields will be sent to the
  1322. // server regardless of whether the field is empty or not. This may be
  1323. // used to include empty fields in Patch requests.
  1324. ForceSendFields []string `json:"-"`
  1325. }
  1326. func (s *SendWorkerMessagesRequest) MarshalJSON() ([]byte, error) {
  1327. type noMethod SendWorkerMessagesRequest
  1328. raw := noMethod(*s)
  1329. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1330. }
  1331. // SendWorkerMessagesResponse: The response to the worker messages.
  1332. type SendWorkerMessagesResponse struct {
  1333. // WorkerMessageResponses: The servers response to the worker messages.
  1334. WorkerMessageResponses []*WorkerMessageResponse `json:"workerMessageResponses,omitempty"`
  1335. // ServerResponse contains the HTTP response code and headers from the
  1336. // server.
  1337. googleapi.ServerResponse `json:"-"`
  1338. // ForceSendFields is a list of field names (e.g.
  1339. // "WorkerMessageResponses") to unconditionally include in API requests.
  1340. // By default, fields with empty values are omitted from API requests.
  1341. // However, any non-pointer, non-interface field appearing in
  1342. // ForceSendFields will be sent to the server regardless of whether the
  1343. // field is empty or not. This may be used to include empty fields in
  1344. // Patch requests.
  1345. ForceSendFields []string `json:"-"`
  1346. }
  1347. func (s *SendWorkerMessagesResponse) MarshalJSON() ([]byte, error) {
  1348. type noMethod SendWorkerMessagesResponse
  1349. raw := noMethod(*s)
  1350. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1351. }
  1352. // SeqMapTask: Describes a particular function to invoke.
  1353. type SeqMapTask struct {
  1354. // Inputs: Information about each of the inputs.
  1355. Inputs []*SideInputInfo `json:"inputs,omitempty"`
  1356. // Name: The user-provided name of the SeqDo operation.
  1357. Name string `json:"name,omitempty"`
  1358. // OutputInfos: Information about each of the outputs.
  1359. OutputInfos []*SeqMapTaskOutputInfo `json:"outputInfos,omitempty"`
  1360. // StageName: System-defined name of the stage containing the SeqDo
  1361. // operation. Unique across the workflow.
  1362. StageName string `json:"stageName,omitempty"`
  1363. // SystemName: System-defined name of the SeqDo operation. Unique across
  1364. // the workflow.
  1365. SystemName string `json:"systemName,omitempty"`
  1366. // UserFn: The user function to invoke.
  1367. UserFn SeqMapTaskUserFn `json:"userFn,omitempty"`
  1368. // ForceSendFields is a list of field names (e.g. "Inputs") to
  1369. // unconditionally include in API requests. By default, fields with
  1370. // empty values are omitted from API requests. However, any non-pointer,
  1371. // non-interface field appearing in ForceSendFields will be sent to the
  1372. // server regardless of whether the field is empty or not. This may be
  1373. // used to include empty fields in Patch requests.
  1374. ForceSendFields []string `json:"-"`
  1375. }
  1376. func (s *SeqMapTask) MarshalJSON() ([]byte, error) {
  1377. type noMethod SeqMapTask
  1378. raw := noMethod(*s)
  1379. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1380. }
  1381. type SeqMapTaskUserFn interface{}
  1382. // SeqMapTaskOutputInfo: Information about an output of a SeqMapTask.
  1383. type SeqMapTaskOutputInfo struct {
  1384. // Sink: The sink to write the output value to.
  1385. Sink *Sink `json:"sink,omitempty"`
  1386. // Tag: The id of the TupleTag the user code will tag the output value
  1387. // by.
  1388. Tag string `json:"tag,omitempty"`
  1389. // ForceSendFields is a list of field names (e.g. "Sink") to
  1390. // unconditionally include in API requests. By default, fields with
  1391. // empty values are omitted from API requests. However, any non-pointer,
  1392. // non-interface field appearing in ForceSendFields will be sent to the
  1393. // server regardless of whether the field is empty or not. This may be
  1394. // used to include empty fields in Patch requests.
  1395. ForceSendFields []string `json:"-"`
  1396. }
  1397. func (s *SeqMapTaskOutputInfo) MarshalJSON() ([]byte, error) {
  1398. type noMethod SeqMapTaskOutputInfo
  1399. raw := noMethod(*s)
  1400. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1401. }
  1402. // ShellTask: A task which consists of a shell command for the worker to
  1403. // execute.
  1404. type ShellTask struct {
  1405. // Command: The shell command to run.
  1406. Command string `json:"command,omitempty"`
  1407. // ExitCode: Exit code for the task.
  1408. ExitCode int64 `json:"exitCode,omitempty"`
  1409. // ForceSendFields is a list of field names (e.g. "Command") to
  1410. // unconditionally include in API requests. By default, fields with
  1411. // empty values are omitted from API requests. However, any non-pointer,
  1412. // non-interface field appearing in ForceSendFields will be sent to the
  1413. // server regardless of whether the field is empty or not. This may be
  1414. // used to include empty fields in Patch requests.
  1415. ForceSendFields []string `json:"-"`
  1416. }
  1417. func (s *ShellTask) MarshalJSON() ([]byte, error) {
  1418. type noMethod ShellTask
  1419. raw := noMethod(*s)
  1420. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1421. }
  1422. // SideInputInfo: Information about a side input of a DoFn or an input
  1423. // of a SeqDoFn.
  1424. type SideInputInfo struct {
  1425. // Kind: How to interpret the source element(s) as a side input value.
  1426. Kind SideInputInfoKind `json:"kind,omitempty"`
  1427. // Sources: The source(s) to read element(s) from to get the value of
  1428. // this side input. If more than one source, then the elements are taken
  1429. // from the sources, in the specified order if order matters. At least
  1430. // one source is required.
  1431. Sources []*Source `json:"sources,omitempty"`
  1432. // Tag: The id of the tag the user code will access this side input by;
  1433. // this should correspond to the tag of some MultiOutputInfo.
  1434. Tag string `json:"tag,omitempty"`
  1435. // ForceSendFields is a list of field names (e.g. "Kind") to
  1436. // unconditionally include in API requests. By default, fields with
  1437. // empty values are omitted from API requests. However, any non-pointer,
  1438. // non-interface field appearing in ForceSendFields will be sent to the
  1439. // server regardless of whether the field is empty or not. This may be
  1440. // used to include empty fields in Patch requests.
  1441. ForceSendFields []string `json:"-"`
  1442. }
  1443. func (s *SideInputInfo) MarshalJSON() ([]byte, error) {
  1444. type noMethod SideInputInfo
  1445. raw := noMethod(*s)
  1446. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1447. }
  1448. type SideInputInfoKind interface{}
  1449. // Sink: A sink that records can be encoded and written to.
  1450. type Sink struct {
  1451. // Codec: The codec to use to encode data written to the sink.
  1452. Codec SinkCodec `json:"codec,omitempty"`
  1453. // Spec: The sink to write to, plus its parameters.
  1454. Spec SinkSpec `json:"spec,omitempty"`
  1455. // ForceSendFields is a list of field names (e.g. "Codec") to
  1456. // unconditionally include in API requests. By default, fields with
  1457. // empty values are omitted from API requests. However, any non-pointer,
  1458. // non-interface field appearing in ForceSendFields will be sent to the
  1459. // server regardless of whether the field is empty or not. This may be
  1460. // used to include empty fields in Patch requests.
  1461. ForceSendFields []string `json:"-"`
  1462. }
  1463. func (s *Sink) MarshalJSON() ([]byte, error) {
  1464. type noMethod Sink
  1465. raw := noMethod(*s)
  1466. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1467. }
  1468. type SinkCodec interface{}
  1469. type SinkSpec interface{}
  1470. // Source: A source that records can be read and decoded from.
  1471. type Source struct {
  1472. // BaseSpecs: While splitting, sources may specify the produced bundles
  1473. // as differences against another source, in order to save backend-side
  1474. // memory and allow bigger jobs. For details, see SourceSplitRequest. To
  1475. // support this use case, the full set of parameters of the source is
  1476. // logically obtained by taking the latest explicitly specified value of
  1477. // each parameter in the order: base_specs (later items win), spec
  1478. // (overrides anything in base_specs).
  1479. BaseSpecs []SourceBaseSpecs `json:"baseSpecs,omitempty"`
  1480. // Codec: The codec to use to decode data read from the source.
  1481. Codec SourceCodec `json:"codec,omitempty"`
  1482. // DoesNotNeedSplitting: Setting this value to true hints to the
  1483. // framework that the source doesn't need splitting, and using
  1484. // SourceSplitRequest on it would yield
  1485. // SOURCE_SPLIT_OUTCOME_USE_CURRENT. E.g. a file splitter may set this
  1486. // to true when splitting a single file into a set of byte ranges of
  1487. // appropriate size, and set this to false when splitting a filepattern
  1488. // into individual files. However, for efficiency, a file splitter may
  1489. // decide to produce file subranges directly from the filepattern to
  1490. // avoid a splitting round-trip. See SourceSplitRequest for an overview
  1491. // of the splitting process. This field is meaningful only in the Source
  1492. // objects populated by the user (e.g. when filling in a DerivedSource).
  1493. // Source objects supplied by the framework to the user don't have this
  1494. // field populated.
  1495. DoesNotNeedSplitting bool `json:"doesNotNeedSplitting,omitempty"`
  1496. // Metadata: Optionally, metadata for this source can be supplied right
  1497. // away, avoiding a SourceGetMetadataOperation roundtrip (see
  1498. // SourceOperationRequest). This field is meaningful only in the Source
  1499. // objects populated by the user (e.g. when filling in a DerivedSource).
  1500. // Source objects supplied by the framework to the user don't have this
  1501. // field populated.
  1502. Metadata *SourceMetadata `json:"metadata,omitempty"`
  1503. // Spec: The source to read from, plus its parameters.
  1504. Spec SourceSpec `json:"spec,omitempty"`
  1505. // ForceSendFields is a list of field names (e.g. "BaseSpecs") to
  1506. // unconditionally include in API requests. By default, fields with
  1507. // empty values are omitted from API requests. However, any non-pointer,
  1508. // non-interface field appearing in ForceSendFields will be sent to the
  1509. // server regardless of whether the field is empty or not. This may be
  1510. // used to include empty fields in Patch requests.
  1511. ForceSendFields []string `json:"-"`
  1512. }
  1513. func (s *Source) MarshalJSON() ([]byte, error) {
  1514. type noMethod Source
  1515. raw := noMethod(*s)
  1516. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1517. }
  1518. type SourceBaseSpecs interface{}
  1519. type SourceCodec interface{}
  1520. type SourceSpec interface{}
  1521. // SourceFork: DEPRECATED in favor of DynamicSourceSplit.
  1522. type SourceFork struct {
  1523. // Primary: DEPRECATED
  1524. Primary *SourceSplitShard `json:"primary,omitempty"`
  1525. // PrimarySource: DEPRECATED
  1526. PrimarySource *DerivedSource `json:"primarySource,omitempty"`
  1527. // Residual: DEPRECATED
  1528. Residual *SourceSplitShard `json:"residual,omitempty"`
  1529. // ResidualSource: DEPRECATED
  1530. ResidualSource *DerivedSource `json:"residualSource,omitempty"`
  1531. // ForceSendFields is a list of field names (e.g. "Primary") to
  1532. // unconditionally include in API requests. By default, fields with
  1533. // empty values are omitted from API requests. However, any non-pointer,
  1534. // non-interface field appearing in ForceSendFields will be sent to the
  1535. // server regardless of whether the field is empty or not. This may be
  1536. // used to include empty fields in Patch requests.
  1537. ForceSendFields []string `json:"-"`
  1538. }
  1539. func (s *SourceFork) MarshalJSON() ([]byte, error) {
  1540. type noMethod SourceFork
  1541. raw := noMethod(*s)
  1542. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1543. }
  1544. // SourceGetMetadataRequest: A request to compute the SourceMetadata of
  1545. // a Source.
  1546. type SourceGetMetadataRequest struct {
  1547. // Source: Specification of the source whose metadata should be
  1548. // computed.
  1549. Source *Source `json:"source,omitempty"`
  1550. // ForceSendFields is a list of field names (e.g. "Source") to
  1551. // unconditionally include in API requests. By default, fields with
  1552. // empty values are omitted from API requests. However, any non-pointer,
  1553. // non-interface field appearing in ForceSendFields will be sent to the
  1554. // server regardless of whether the field is empty or not. This may be
  1555. // used to include empty fields in Patch requests.
  1556. ForceSendFields []string `json:"-"`
  1557. }
  1558. func (s *SourceGetMetadataRequest) MarshalJSON() ([]byte, error) {
  1559. type noMethod SourceGetMetadataRequest
  1560. raw := noMethod(*s)
  1561. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1562. }
  1563. // SourceGetMetadataResponse: The result of a
  1564. // SourceGetMetadataOperation.
  1565. type SourceGetMetadataResponse struct {
  1566. // Metadata: The computed metadata.
  1567. Metadata *SourceMetadata `json:"metadata,omitempty"`
  1568. // ForceSendFields is a list of field names (e.g. "Metadata") to
  1569. // unconditionally include in API requests. By default, fields with
  1570. // empty values are omitted from API requests. However, any non-pointer,
  1571. // non-interface field appearing in ForceSendFields will be sent to the
  1572. // server regardless of whether the field is empty or not. This may be
  1573. // used to include empty fields in Patch requests.
  1574. ForceSendFields []string `json:"-"`
  1575. }
  1576. func (s *SourceGetMetadataResponse) MarshalJSON() ([]byte, error) {
  1577. type noMethod SourceGetMetadataResponse
  1578. raw := noMethod(*s)
  1579. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1580. }
  1581. // SourceMetadata: Metadata about a Source useful for automatically
  1582. // optimizing and tuning the pipeline, etc.
  1583. type SourceMetadata struct {
  1584. // EstimatedSizeBytes: An estimate of the total size (in bytes) of the
  1585. // data that would be read from this source. This estimate is in terms
  1586. // of external storage size, before any decompression or other
  1587. // processing done by the reader.
  1588. EstimatedSizeBytes int64 `json:"estimatedSizeBytes,omitempty,string"`
  1589. // Infinite: Specifies that the size of this source is known to be
  1590. // infinite (this is a streaming source).
  1591. Infinite bool `json:"infinite,omitempty"`
  1592. // ProducesSortedKeys: Whether this source is known to produce key/value
  1593. // pairs with the (encoded) keys in lexicographically sorted order.
  1594. ProducesSortedKeys bool `json:"producesSortedKeys,omitempty"`
  1595. // ForceSendFields is a list of field names (e.g. "EstimatedSizeBytes")
  1596. // to unconditionally include in API requests. By default, fields with
  1597. // empty values are omitted from API requests. However, any non-pointer,
  1598. // non-interface field appearing in ForceSendFields will be sent to the
  1599. // server regardless of whether the field is empty or not. This may be
  1600. // used to include empty fields in Patch requests.
  1601. ForceSendFields []string `json:"-"`
  1602. }
  1603. func (s *SourceMetadata) MarshalJSON() ([]byte, error) {
  1604. type noMethod SourceMetadata
  1605. raw := noMethod(*s)
  1606. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1607. }
  1608. // SourceOperationRequest: A work item that represents the different
  1609. // operations that can be performed on a user-defined Source
  1610. // specification.
  1611. type SourceOperationRequest struct {
  1612. // GetMetadata: Information about a request to get metadata about a
  1613. // source.
  1614. GetMetadata *SourceGetMetadataRequest `json:"getMetadata,omitempty"`
  1615. // Split: Information about a request to split a source.
  1616. Split *SourceSplitRequest `json:"split,omitempty"`
  1617. // ForceSendFields is a list of field names (e.g. "GetMetadata") to
  1618. // unconditionally include in API requests. By default, fields with
  1619. // empty values are omitted from API requests. However, any non-pointer,
  1620. // non-interface field appearing in ForceSendFields will be sent to the
  1621. // server regardless of whether the field is empty or not. This may be
  1622. // used to include empty fields in Patch requests.
  1623. ForceSendFields []string `json:"-"`
  1624. }
  1625. func (s *SourceOperationRequest) MarshalJSON() ([]byte, error) {
  1626. type noMethod SourceOperationRequest
  1627. raw := noMethod(*s)
  1628. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1629. }
  1630. // SourceOperationResponse: The result of a SourceOperationRequest,
  1631. // specified in ReportWorkItemStatusRequest.source_operation when the
  1632. // work item is completed.
  1633. type SourceOperationResponse struct {
  1634. // GetMetadata: A response to a request to get metadata about a source.
  1635. GetMetadata *SourceGetMetadataResponse `json:"getMetadata,omitempty"`
  1636. // Split: A response to a request to split a source.
  1637. Split *SourceSplitResponse `json:"split,omitempty"`
  1638. // ForceSendFields is a list of field names (e.g. "GetMetadata") to
  1639. // unconditionally include in API requests. By default, fields with
  1640. // empty values are omitted from API requests. However, any non-pointer,
  1641. // non-interface field appearing in ForceSendFields will be sent to the
  1642. // server regardless of whether the field is empty or not. This may be
  1643. // used to include empty fields in Patch requests.
  1644. ForceSendFields []string `json:"-"`
  1645. }
  1646. func (s *SourceOperationResponse) MarshalJSON() ([]byte, error) {
  1647. type noMethod SourceOperationResponse
  1648. raw := noMethod(*s)
  1649. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1650. }
  1651. // SourceSplitOptions: Hints for splitting a Source into bundles (parts
  1652. // for parallel processing) using SourceSplitRequest.
  1653. type SourceSplitOptions struct {
  1654. // DesiredBundleSizeBytes: The source should be split into a set of
  1655. // bundles where the estimated size of each is approximately this many
  1656. // bytes.
  1657. DesiredBundleSizeBytes int64 `json:"desiredBundleSizeBytes,omitempty,string"`
  1658. // DesiredShardSizeBytes: DEPRECATED in favor of
  1659. // desired_bundle_size_bytes.
  1660. DesiredShardSizeBytes int64 `json:"desiredShardSizeBytes,omitempty,string"`
  1661. // ForceSendFields is a list of field names (e.g.
  1662. // "DesiredBundleSizeBytes") to unconditionally include in API requests.
  1663. // By default, fields with empty values are omitted from API requests.
  1664. // However, any non-pointer, non-interface field appearing in
  1665. // ForceSendFields will be sent to the server regardless of whether the
  1666. // field is empty or not. This may be used to include empty fields in
  1667. // Patch requests.
  1668. ForceSendFields []string `json:"-"`
  1669. }
  1670. func (s *SourceSplitOptions) MarshalJSON() ([]byte, error) {
  1671. type noMethod SourceSplitOptions
  1672. raw := noMethod(*s)
  1673. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1674. }
  1675. // SourceSplitRequest: Represents the operation to split a high-level
  1676. // Source specification into bundles (parts for parallel processing). At
  1677. // a high level, splitting of a source into bundles happens as follows:
  1678. // SourceSplitRequest is applied to the source. If it returns
  1679. // SOURCE_SPLIT_OUTCOME_USE_CURRENT, no further splitting happens and
  1680. // the source is used "as is". Otherwise, splitting is applied
  1681. // recursively to each produced DerivedSource. As an optimization, for
  1682. // any Source, if its does_not_need_splitting is true, the framework
  1683. // assumes that splitting this source would return
  1684. // SOURCE_SPLIT_OUTCOME_USE_CURRENT, and doesn't initiate a
  1685. // SourceSplitRequest. This applies both to the initial source being
  1686. // split and to bundles produced from it.
  1687. type SourceSplitRequest struct {
  1688. // Options: Hints for tuning the splitting process.
  1689. Options *SourceSplitOptions `json:"options,omitempty"`
  1690. // Source: Specification of the source to be split.
  1691. Source *Source `json:"source,omitempty"`
  1692. // ForceSendFields is a list of field names (e.g. "Options") to
  1693. // unconditionally include in API requests. By default, fields with
  1694. // empty values are omitted from API requests. However, any non-pointer,
  1695. // non-interface field appearing in ForceSendFields will be sent to the
  1696. // server regardless of whether the field is empty or not. This may be
  1697. // used to include empty fields in Patch requests.
  1698. ForceSendFields []string `json:"-"`
  1699. }
  1700. func (s *SourceSplitRequest) MarshalJSON() ([]byte, error) {
  1701. type noMethod SourceSplitRequest
  1702. raw := noMethod(*s)
  1703. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1704. }
  1705. // SourceSplitResponse: The response to a SourceSplitRequest.
  1706. type SourceSplitResponse struct {
  1707. // Bundles: If outcome is SPLITTING_HAPPENED, then this is a list of
  1708. // bundles into which the source was split. Otherwise this field is
  1709. // ignored. This list can be empty, which means the source represents an
  1710. // empty input.
  1711. Bundles []*DerivedSource `json:"bundles,omitempty"`
  1712. // Outcome: Indicates whether splitting happened and produced a list of
  1713. // bundles. If this is USE_CURRENT_SOURCE_AS_IS, the current source
  1714. // should be processed "as is" without splitting. "bundles" is ignored
  1715. // in this case. If this is SPLITTING_HAPPENED, then "bundles" contains
  1716. // a list of bundles into which the source was split.
  1717. //
  1718. // Possible values:
  1719. // "SOURCE_SPLIT_OUTCOME_UNKNOWN"
  1720. // "SOURCE_SPLIT_OUTCOME_USE_CURRENT"
  1721. // "SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED"
  1722. Outcome string `json:"outcome,omitempty"`
  1723. // Shards: DEPRECATED in favor of bundles.
  1724. Shards []*SourceSplitShard `json:"shards,omitempty"`
  1725. // ForceSendFields is a list of field names (e.g. "Bundles") to
  1726. // unconditionally include in API requests. By default, fields with
  1727. // empty values are omitted from API requests. However, any non-pointer,
  1728. // non-interface field appearing in ForceSendFields will be sent to the
  1729. // server regardless of whether the field is empty or not. This may be
  1730. // used to include empty fields in Patch requests.
  1731. ForceSendFields []string `json:"-"`
  1732. }
  1733. func (s *SourceSplitResponse) MarshalJSON() ([]byte, error) {
  1734. type noMethod SourceSplitResponse
  1735. raw := noMethod(*s)
  1736. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1737. }
  1738. // SourceSplitShard: DEPRECATED in favor of DerivedSource.
  1739. type SourceSplitShard struct {
  1740. // DerivationMode: DEPRECATED
  1741. //
  1742. // Possible values:
  1743. // "SOURCE_DERIVATION_MODE_UNKNOWN"
  1744. // "SOURCE_DERIVATION_MODE_INDEPENDENT"
  1745. // "SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT"
  1746. // "SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT"
  1747. DerivationMode string `json:"derivationMode,omitempty"`
  1748. // Source: DEPRECATED
  1749. Source *Source `json:"source,omitempty"`
  1750. // ForceSendFields is a list of field names (e.g. "DerivationMode") to
  1751. // unconditionally include in API requests. By default, fields with
  1752. // empty values are omitted from API requests. However, any non-pointer,
  1753. // non-interface field appearing in ForceSendFields will be sent to the
  1754. // server regardless of whether the field is empty or not. This may be
  1755. // used to include empty fields in Patch requests.
  1756. ForceSendFields []string `json:"-"`
  1757. }
  1758. func (s *SourceSplitShard) MarshalJSON() ([]byte, error) {
  1759. type noMethod SourceSplitShard
  1760. raw := noMethod(*s)
  1761. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1762. }
  1763. // StateFamilyConfig: State family configuration.
  1764. type StateFamilyConfig struct {
  1765. // IsRead: If true, this family corresponds to a read operation.
  1766. IsRead bool `json:"isRead,omitempty"`
  1767. // StateFamily: The state family value.
  1768. StateFamily string `json:"stateFamily,omitempty"`
  1769. // ForceSendFields is a list of field names (e.g. "IsRead") to
  1770. // unconditionally include in API requests. By default, fields with
  1771. // empty values are omitted from API requests. However, any non-pointer,
  1772. // non-interface field appearing in ForceSendFields will be sent to the
  1773. // server regardless of whether the field is empty or not. This may be
  1774. // used to include empty fields in Patch requests.
  1775. ForceSendFields []string `json:"-"`
  1776. }
  1777. func (s *StateFamilyConfig) MarshalJSON() ([]byte, error) {
  1778. type noMethod StateFamilyConfig
  1779. raw := noMethod(*s)
  1780. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1781. }
  1782. // Status: The `Status` type defines a logical error model that is
  1783. // suitable for different programming environments, including REST APIs
  1784. // and RPC APIs. It is used by [gRPC](https://github.com/grpc). The
  1785. // error model is designed to be: - Simple to use and understand for
  1786. // most users - Flexible enough to meet unexpected needs # Overview The
  1787. // `Status` message contains three pieces of data: error code, error
  1788. // message, and error details. The error code should be an enum value of
  1789. // google.rpc.Code, but it may accept additional error codes if needed.
  1790. // The error message should be a developer-facing English message that
  1791. // helps developers *understand* and *resolve* the error. If a localized
  1792. // user-facing error message is needed, put the localized message in the
  1793. // error details or localize it in the client. The optional error
  1794. // details may contain arbitrary information about the error. There is a
  1795. // predefined set of error detail types in the package `google.rpc`
  1796. // which can be used for common error conditions. # Language mapping The
  1797. // `Status` message is the logical representation of the error model,
  1798. // but it is not necessarily the actual wire format. When the `Status`
  1799. // message is exposed in different client libraries and different wire
  1800. // protocols, it can be mapped differently. For example, it will likely
  1801. // be mapped to some exceptions in Java, but more likely mapped to some
  1802. // error codes in C. # Other uses The error model and the `Status`
  1803. // message can be used in a variety of environments, either with or
  1804. // without APIs, to provide a consistent developer experience across
  1805. // different environments. Example uses of this error model include: -
  1806. // Partial errors. If a service needs to return partial errors to the
  1807. // client, it may embed the `Status` in the normal response to indicate
  1808. // the partial errors. - Workflow errors. A typical workflow has
  1809. // multiple steps. Each step may have a `Status` message for error
  1810. // reporting purpose. - Batch operations. If a client uses batch request
  1811. // and batch response, the `Status` message should be used directly
  1812. // inside batch response, one for each error sub-response. -
  1813. // Asynchronous operations. If an API call embeds asynchronous operation
  1814. // results in its response, the status of those operations should be
  1815. // represented directly using the `Status` message. - Logging. If some
  1816. // API errors are stored in logs, the message `Status` could be used
  1817. // directly after any stripping needed for security/privacy reasons.
  1818. type Status struct {
  1819. // Code: The status code, which should be an enum value of
  1820. // google.rpc.Code.
  1821. Code int64 `json:"code,omitempty"`
  1822. // Details: A list of messages that carry the error details. There will
  1823. // be a common set of message types for APIs to use.
  1824. Details []StatusDetails `json:"details,omitempty"`
  1825. // Message: A developer-facing error message, which should be in
  1826. // English. Any user-facing error message should be localized and sent
  1827. // in the google.rpc.Status.details field, or localized by the client.
  1828. Message string `json:"message,omitempty"`
  1829. // ForceSendFields is a list of field names (e.g. "Code") to
  1830. // unconditionally include in API requests. By default, fields with
  1831. // empty values are omitted from API requests. However, any non-pointer,
  1832. // non-interface field appearing in ForceSendFields will be sent to the
  1833. // server regardless of whether the field is empty or not. This may be
  1834. // used to include empty fields in Patch requests.
  1835. ForceSendFields []string `json:"-"`
  1836. }
  1837. func (s *Status) MarshalJSON() ([]byte, error) {
  1838. type noMethod Status
  1839. raw := noMethod(*s)
  1840. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1841. }
  1842. type StatusDetails interface{}
  1843. // Step: Defines a particular step within a Dataflow job. A job consists
  1844. // of multiple steps, each of which performs some specific operation as
  1845. // part of the overall job. Data is typically passed from one step to
  1846. // another as part of the job. Here's an example of a sequence of steps
  1847. // which together implement a Map-Reduce job: * Read a collection of
  1848. // data from some source, parsing the collection's elements. * Validate
  1849. // the elements. * Apply a user-defined function to map each element to
  1850. // some value and extract an element-specific key value. * Group
  1851. // elements with the same key into a single element with that key,
  1852. // transforming a multiply-keyed collection into a uniquely-keyed
  1853. // collection. * Write the elements out to some data sink. (Note that
  1854. // the Dataflow service may be used to run many different types of jobs,
  1855. // not just Map-Reduce).
  1856. type Step struct {
  1857. // Kind: The kind of step in the dataflow Job.
  1858. Kind string `json:"kind,omitempty"`
  1859. // Name: Name identifying the step. This must be unique for each step
  1860. // with respect to all other steps in the dataflow Job.
  1861. Name string `json:"name,omitempty"`
  1862. // Properties: Named properties associated with the step. Each kind of
  1863. // predefined step has its own required set of properties.
  1864. Properties StepProperties `json:"properties,omitempty"`
  1865. // ForceSendFields is a list of field names (e.g. "Kind") to
  1866. // unconditionally include in API requests. By default, fields with
  1867. // empty values are omitted from API requests. However, any non-pointer,
  1868. // non-interface field appearing in ForceSendFields will be sent to the
  1869. // server regardless of whether the field is empty or not. This may be
  1870. // used to include empty fields in Patch requests.
  1871. ForceSendFields []string `json:"-"`
  1872. }
  1873. func (s *Step) MarshalJSON() ([]byte, error) {
  1874. type noMethod Step
  1875. raw := noMethod(*s)
  1876. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1877. }
  1878. type StepProperties interface{}
  1879. // StreamLocation: Describes a stream of data, either as input to be
  1880. // processed or as output of a streaming Dataflow job.
  1881. type StreamLocation struct {
  1882. // CustomSourceLocation: The stream is a custom source.
  1883. CustomSourceLocation *CustomSourceLocation `json:"customSourceLocation,omitempty"`
  1884. // PubsubLocation: The stream is a pubsub stream.
  1885. PubsubLocation *PubsubLocation `json:"pubsubLocation,omitempty"`
  1886. // SideInputLocation: The stream is a streaming side input.
  1887. SideInputLocation *StreamingSideInputLocation `json:"sideInputLocation,omitempty"`
  1888. // StreamingStageLocation: The stream is part of another computation
  1889. // within the current streaming Dataflow job.
  1890. StreamingStageLocation *StreamingStageLocation `json:"streamingStageLocation,omitempty"`
  1891. // ForceSendFields is a list of field names (e.g.
  1892. // "CustomSourceLocation") to unconditionally include in API requests.
  1893. // By default, fields with empty values are omitted from API requests.
  1894. // However, any non-pointer, non-interface field appearing in
  1895. // ForceSendFields will be sent to the server regardless of whether the
  1896. // field is empty or not. This may be used to include empty fields in
  1897. // Patch requests.
  1898. ForceSendFields []string `json:"-"`
  1899. }
  1900. func (s *StreamLocation) MarshalJSON() ([]byte, error) {
  1901. type noMethod StreamLocation
  1902. raw := noMethod(*s)
  1903. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1904. }
  1905. // StreamingComputationRanges: Describes full or partial data disk
  1906. // assignment information of the computation ranges.
  1907. type StreamingComputationRanges struct {
  1908. // ComputationId: The ID of the computation.
  1909. ComputationId string `json:"computationId,omitempty"`
  1910. // RangeAssignments: Data disk assignments for ranges from this
  1911. // computation.
  1912. RangeAssignments []*KeyRangeDataDiskAssignment `json:"rangeAssignments,omitempty"`
  1913. // ForceSendFields is a list of field names (e.g. "ComputationId") to
  1914. // unconditionally include in API requests. By default, fields with
  1915. // empty values are omitted from API requests. However, any non-pointer,
  1916. // non-interface field appearing in ForceSendFields will be sent to the
  1917. // server regardless of whether the field is empty or not. This may be
  1918. // used to include empty fields in Patch requests.
  1919. ForceSendFields []string `json:"-"`
  1920. }
  1921. func (s *StreamingComputationRanges) MarshalJSON() ([]byte, error) {
  1922. type noMethod StreamingComputationRanges
  1923. raw := noMethod(*s)
  1924. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1925. }
  1926. // StreamingComputationTask: A task which describes what action should
  1927. // be performed for the specified streaming computation ranges.
  1928. type StreamingComputationTask struct {
  1929. // ComputationRanges: Contains ranges of a streaming computation this
  1930. // task should apply to.
  1931. ComputationRanges []*StreamingComputationRanges `json:"computationRanges,omitempty"`
  1932. // DataDisks: Describes the set of data disks this task should apply to.
  1933. DataDisks []*MountedDataDisk `json:"dataDisks,omitempty"`
  1934. // TaskType: A type of streaming computation task.
  1935. //
  1936. // Possible values:
  1937. // "STREAMING_COMPUTATION_TASK_UNKNOWN"
  1938. // "STREAMING_COMPUTATION_TASK_STOP"
  1939. // "STREAMING_COMPUTATION_TASK_START"
  1940. TaskType string `json:"taskType,omitempty"`
  1941. // ForceSendFields is a list of field names (e.g. "ComputationRanges")
  1942. // to unconditionally include in API requests. By default, fields with
  1943. // empty values are omitted from API requests. However, any non-pointer,
  1944. // non-interface field appearing in ForceSendFields will be sent to the
  1945. // server regardless of whether the field is empty or not. This may be
  1946. // used to include empty fields in Patch requests.
  1947. ForceSendFields []string `json:"-"`
  1948. }
  1949. func (s *StreamingComputationTask) MarshalJSON() ([]byte, error) {
  1950. type noMethod StreamingComputationTask
  1951. raw := noMethod(*s)
  1952. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1953. }
  1954. // StreamingSetupTask: A task which initializes part of a streaming
  1955. // Dataflow job.
  1956. type StreamingSetupTask struct {
  1957. // ReceiveWorkPort: The TCP port on which the worker should listen for
  1958. // messages from other streaming computation workers.
  1959. ReceiveWorkPort int64 `json:"receiveWorkPort,omitempty"`
  1960. // StreamingComputationTopology: The global topology of the streaming
  1961. // Dataflow job.
  1962. StreamingComputationTopology *TopologyConfig `json:"streamingComputationTopology,omitempty"`
  1963. // WorkerHarnessPort: The TCP port used by the worker to communicate
  1964. // with the Dataflow worker harness.
  1965. WorkerHarnessPort int64 `json:"workerHarnessPort,omitempty"`
  1966. // ForceSendFields is a list of field names (e.g. "ReceiveWorkPort") to
  1967. // unconditionally include in API requests. By default, fields with
  1968. // empty values are omitted from API requests. However, any non-pointer,
  1969. // non-interface field appearing in ForceSendFields will be sent to the
  1970. // server regardless of whether the field is empty or not. This may be
  1971. // used to include empty fields in Patch requests.
  1972. ForceSendFields []string `json:"-"`
  1973. }
  1974. func (s *StreamingSetupTask) MarshalJSON() ([]byte, error) {
  1975. type noMethod StreamingSetupTask
  1976. raw := noMethod(*s)
  1977. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  1978. }
  1979. // StreamingSideInputLocation: Identifies the location of a streaming
  1980. // side input.
  1981. type StreamingSideInputLocation struct {
  1982. // StateFamily: Identifies the state family where this side input is
  1983. // stored.
  1984. StateFamily string `json:"stateFamily,omitempty"`
  1985. // Tag: Identifies the particular side input within the streaming
  1986. // Dataflow job.
  1987. Tag string `json:"tag,omitempty"`
  1988. // ForceSendFields is a list of field names (e.g. "StateFamily") to
  1989. // unconditionally include in API requests. By default, fields with
  1990. // empty values are omitted from API requests. However, any non-pointer,
  1991. // non-interface field appearing in ForceSendFields will be sent to the
  1992. // server regardless of whether the field is empty or not. This may be
  1993. // used to include empty fields in Patch requests.
  1994. ForceSendFields []string `json:"-"`
  1995. }
  1996. func (s *StreamingSideInputLocation) MarshalJSON() ([]byte, error) {
  1997. type noMethod StreamingSideInputLocation
  1998. raw := noMethod(*s)
  1999. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2000. }
  2001. // StreamingStageLocation: Identifies the location of a streaming
  2002. // computation stage, for stage-to-stage communication.
  2003. type StreamingStageLocation struct {
  2004. // StreamId: Identifies the particular stream within the streaming
  2005. // Dataflow job.
  2006. StreamId string `json:"streamId,omitempty"`
  2007. // ForceSendFields is a list of field names (e.g. "StreamId") to
  2008. // unconditionally include in API requests. By default, fields with
  2009. // empty values are omitted from API requests. However, any non-pointer,
  2010. // non-interface field appearing in ForceSendFields will be sent to the
  2011. // server regardless of whether the field is empty or not. This may be
  2012. // used to include empty fields in Patch requests.
  2013. ForceSendFields []string `json:"-"`
  2014. }
  2015. func (s *StreamingStageLocation) MarshalJSON() ([]byte, error) {
  2016. type noMethod StreamingStageLocation
  2017. raw := noMethod(*s)
  2018. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2019. }
  2020. // TaskRunnerSettings: Taskrunner configuration settings.
  2021. type TaskRunnerSettings struct {
  2022. // Alsologtostderr: Also send taskrunner log info to stderr?
  2023. Alsologtostderr bool `json:"alsologtostderr,omitempty"`
  2024. // BaseTaskDir: Location on the worker for task-specific subdirectories.
  2025. BaseTaskDir string `json:"baseTaskDir,omitempty"`
  2026. // BaseUrl: The base URL for the taskrunner to use when accessing Google
  2027. // Cloud APIs. When workers access Google Cloud APIs, they logically do
  2028. // so via relative URLs. If this field is specified, it supplies the
  2029. // base URL to use for resolving these relative URLs. The normative
  2030. // algorithm used is defined by RFC 1808, "Relative Uniform Resource
  2031. // Locators". If not specified, the default value is
  2032. // "http://www.googleapis.com/"
  2033. BaseUrl string `json:"baseUrl,omitempty"`
  2034. // CommandlinesFileName: Store preprocessing commands in this file.
  2035. CommandlinesFileName string `json:"commandlinesFileName,omitempty"`
  2036. // ContinueOnException: Do we continue taskrunner if an exception is
  2037. // hit?
  2038. ContinueOnException bool `json:"continueOnException,omitempty"`
  2039. // DataflowApiVersion: API version of endpoint, e.g. "v1b3"
  2040. DataflowApiVersion string `json:"dataflowApiVersion,omitempty"`
  2041. // HarnessCommand: Command to launch the worker harness.
  2042. HarnessCommand string `json:"harnessCommand,omitempty"`
  2043. // LanguageHint: Suggested backend language.
  2044. LanguageHint string `json:"languageHint,omitempty"`
  2045. // LogDir: Directory on the VM to store logs.
  2046. LogDir string `json:"logDir,omitempty"`
  2047. // LogToSerialconsole: Send taskrunner log into to Google Compute Engine
  2048. // VM serial console?
  2049. LogToSerialconsole bool `json:"logToSerialconsole,omitempty"`
  2050. // LogUploadLocation: Indicates where to put logs. If this is not
  2051. // specified, the logs will not be uploaded. The supported resource type
  2052. // is: Google Cloud Storage: storage.googleapis.com/{bucket}/{object}
  2053. // bucket.storage.googleapis.com/{object}
  2054. LogUploadLocation string `json:"logUploadLocation,omitempty"`
  2055. // OauthScopes: OAuth2 scopes to be requested by the taskrunner in order
  2056. // to access the dataflow API.
  2057. OauthScopes []string `json:"oauthScopes,omitempty"`
  2058. // ParallelWorkerSettings: Settings to pass to the parallel worker
  2059. // harness.
  2060. ParallelWorkerSettings *WorkerSettings `json:"parallelWorkerSettings,omitempty"`
  2061. // StreamingWorkerMainClass: Streaming worker main class name.
  2062. StreamingWorkerMainClass string `json:"streamingWorkerMainClass,omitempty"`
  2063. // TaskGroup: The UNIX group ID on the worker VM to use for tasks
  2064. // launched by taskrunner; e.g. "wheel".
  2065. TaskGroup string `json:"taskGroup,omitempty"`
  2066. // TaskUser: The UNIX user ID on the worker VM to use for tasks launched
  2067. // by taskrunner; e.g. "root".
  2068. TaskUser string `json:"taskUser,omitempty"`
  2069. // TempStoragePrefix: The prefix of the resources the taskrunner should
  2070. // use for temporary storage. The supported resource type is: Google
  2071. // Cloud Storage: storage.googleapis.com/{bucket}/{object}
  2072. // bucket.storage.googleapis.com/{object}
  2073. TempStoragePrefix string `json:"tempStoragePrefix,omitempty"`
  2074. // VmId: ID string of VM.
  2075. VmId string `json:"vmId,omitempty"`
  2076. // WorkflowFileName: Store the workflow in this file.
  2077. WorkflowFileName string `json:"workflowFileName,omitempty"`
  2078. // ForceSendFields is a list of field names (e.g. "Alsologtostderr") to
  2079. // unconditionally include in API requests. By default, fields with
  2080. // empty values are omitted from API requests. However, any non-pointer,
  2081. // non-interface field appearing in ForceSendFields will be sent to the
  2082. // server regardless of whether the field is empty or not. This may be
  2083. // used to include empty fields in Patch requests.
  2084. ForceSendFields []string `json:"-"`
  2085. }
  2086. func (s *TaskRunnerSettings) MarshalJSON() ([]byte, error) {
  2087. type noMethod TaskRunnerSettings
  2088. raw := noMethod(*s)
  2089. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2090. }
  2091. // TopologyConfig: Global topology of the streaming Dataflow job,
  2092. // including all computations and their sharded locations.
  2093. type TopologyConfig struct {
  2094. // Computations: The computations associated with a streaming Dataflow
  2095. // job.
  2096. Computations []*ComputationTopology `json:"computations,omitempty"`
  2097. // DataDiskAssignments: The disks assigned to a streaming Dataflow job.
  2098. DataDiskAssignments []*DataDiskAssignment `json:"dataDiskAssignments,omitempty"`
  2099. // ForwardingKeyBits: The size (in bits) of keys that will be assigned
  2100. // to source messages.
  2101. ForwardingKeyBits int64 `json:"forwardingKeyBits,omitempty"`
  2102. // UserStageToComputationNameMap: Maps user stage names to stable
  2103. // computation names.
  2104. UserStageToComputationNameMap map[string]string `json:"userStageToComputationNameMap,omitempty"`
  2105. // ForceSendFields is a list of field names (e.g. "Computations") to
  2106. // unconditionally include in API requests. By default, fields with
  2107. // empty values are omitted from API requests. However, any non-pointer,
  2108. // non-interface field appearing in ForceSendFields will be sent to the
  2109. // server regardless of whether the field is empty or not. This may be
  2110. // used to include empty fields in Patch requests.
  2111. ForceSendFields []string `json:"-"`
  2112. }
  2113. func (s *TopologyConfig) MarshalJSON() ([]byte, error) {
  2114. type noMethod TopologyConfig
  2115. raw := noMethod(*s)
  2116. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2117. }
  2118. // WorkItem: WorkItem represents basic information about a WorkItem to
  2119. // be executed in the cloud.
  2120. type WorkItem struct {
  2121. // Configuration: Work item-specific configuration as an opaque blob.
  2122. Configuration string `json:"configuration,omitempty"`
  2123. // Id: Identifies this WorkItem.
  2124. Id int64 `json:"id,omitempty,string"`
  2125. // InitialReportIndex: The initial index to use when reporting the
  2126. // status of the WorkItem.
  2127. InitialReportIndex int64 `json:"initialReportIndex,omitempty,string"`
  2128. // JobId: Identifies the workflow job this WorkItem belongs to.
  2129. JobId string `json:"jobId,omitempty"`
  2130. // LeaseExpireTime: Time when the lease on this Work will expire.
  2131. LeaseExpireTime string `json:"leaseExpireTime,omitempty"`
  2132. // MapTask: Additional information for MapTask WorkItems.
  2133. MapTask *MapTask `json:"mapTask,omitempty"`
  2134. // Packages: Any required packages that need to be fetched in order to
  2135. // execute this WorkItem.
  2136. Packages []*Package `json:"packages,omitempty"`
  2137. // ProjectId: Identifies the cloud project this WorkItem belongs to.
  2138. ProjectId string `json:"projectId,omitempty"`
  2139. // ReportStatusInterval: Recommended reporting interval.
  2140. ReportStatusInterval string `json:"reportStatusInterval,omitempty"`
  2141. // SeqMapTask: Additional information for SeqMapTask WorkItems.
  2142. SeqMapTask *SeqMapTask `json:"seqMapTask,omitempty"`
  2143. // ShellTask: Additional information for ShellTask WorkItems.
  2144. ShellTask *ShellTask `json:"shellTask,omitempty"`
  2145. // SourceOperationTask: Additional information for source operation
  2146. // WorkItems.
  2147. SourceOperationTask *SourceOperationRequest `json:"sourceOperationTask,omitempty"`
  2148. // StreamingComputationTask: Additional information for
  2149. // StreamingComputationTask WorkItems.
  2150. StreamingComputationTask *StreamingComputationTask `json:"streamingComputationTask,omitempty"`
  2151. // StreamingSetupTask: Additional information for StreamingSetupTask
  2152. // WorkItems.
  2153. StreamingSetupTask *StreamingSetupTask `json:"streamingSetupTask,omitempty"`
  2154. // ForceSendFields is a list of field names (e.g. "Configuration") to
  2155. // unconditionally include in API requests. By default, fields with
  2156. // empty values are omitted from API requests. However, any non-pointer,
  2157. // non-interface field appearing in ForceSendFields will be sent to the
  2158. // server regardless of whether the field is empty or not. This may be
  2159. // used to include empty fields in Patch requests.
  2160. ForceSendFields []string `json:"-"`
  2161. }
  2162. func (s *WorkItem) MarshalJSON() ([]byte, error) {
  2163. type noMethod WorkItem
  2164. raw := noMethod(*s)
  2165. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2166. }
  2167. // WorkItemServiceState: The Dataflow service's idea of the current
  2168. // state of a WorkItem being processed by a worker.
  2169. type WorkItemServiceState struct {
  2170. // HarnessData: Other data returned by the service, specific to the
  2171. // particular worker harness.
  2172. HarnessData WorkItemServiceStateHarnessData `json:"harnessData,omitempty"`
  2173. // LeaseExpireTime: Time at which the current lease will expire.
  2174. LeaseExpireTime string `json:"leaseExpireTime,omitempty"`
  2175. // NextReportIndex: The index value to use for the next report sent by
  2176. // the worker. Note: If the report call fails for whatever reason, the
  2177. // worker should reuse this index for subsequent report attempts.
  2178. NextReportIndex int64 `json:"nextReportIndex,omitempty,string"`
  2179. // ReportStatusInterval: New recommended reporting interval.
  2180. ReportStatusInterval string `json:"reportStatusInterval,omitempty"`
  2181. // SplitRequest: The progress point in the WorkItem where the Dataflow
  2182. // service suggests that the worker truncate the task.
  2183. SplitRequest *ApproximateSplitRequest `json:"splitRequest,omitempty"`
  2184. // SuggestedStopPoint: DEPRECATED in favor of split_request.
  2185. SuggestedStopPoint *ApproximateProgress `json:"suggestedStopPoint,omitempty"`
  2186. // SuggestedStopPosition: Obsolete, always empty.
  2187. SuggestedStopPosition *Position `json:"suggestedStopPosition,omitempty"`
  2188. // ForceSendFields is a list of field names (e.g. "HarnessData") to
  2189. // unconditionally include in API requests. By default, fields with
  2190. // empty values are omitted from API requests. However, any non-pointer,
  2191. // non-interface field appearing in ForceSendFields will be sent to the
  2192. // server regardless of whether the field is empty or not. This may be
  2193. // used to include empty fields in Patch requests.
  2194. ForceSendFields []string `json:"-"`
  2195. }
  2196. func (s *WorkItemServiceState) MarshalJSON() ([]byte, error) {
  2197. type noMethod WorkItemServiceState
  2198. raw := noMethod(*s)
  2199. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2200. }
  2201. type WorkItemServiceStateHarnessData interface{}
  2202. // WorkItemStatus: Conveys a worker's progress through the work
  2203. // described by a WorkItem.
  2204. type WorkItemStatus struct {
  2205. // Completed: True if the WorkItem was completed (successfully or
  2206. // unsuccessfully).
  2207. Completed bool `json:"completed,omitempty"`
  2208. // DynamicSourceSplit: See documentation of stop_position.
  2209. DynamicSourceSplit *DynamicSourceSplit `json:"dynamicSourceSplit,omitempty"`
  2210. // Errors: Specifies errors which occurred during processing. If errors
  2211. // are provided, and completed = true, then the WorkItem is considered
  2212. // to have failed.
  2213. Errors []*Status `json:"errors,omitempty"`
  2214. // MetricUpdates: Worker output metrics (counters) for this WorkItem.
  2215. MetricUpdates []*MetricUpdate `json:"metricUpdates,omitempty"`
  2216. // Progress: DEPRECATED in favor of reported_progress.
  2217. Progress *ApproximateProgress `json:"progress,omitempty"`
  2218. // ReportIndex: The report index. When a WorkItem is leased, the lease
  2219. // will contain an initial report index. When a WorkItem's status is
  2220. // reported to the system, the report should be sent with that report
  2221. // index, and the response will contain the index the worker should use
  2222. // for the next report. Reports received with unexpected index values
  2223. // will be rejected by the service. In order to preserve idempotency,
  2224. // the worker should not alter the contents of a report, even if the
  2225. // worker must submit the same report multiple times before getting back
  2226. // a response. The worker should not submit a subsequent report until
  2227. // the response for the previous report had been received from the
  2228. // service.
  2229. ReportIndex int64 `json:"reportIndex,omitempty,string"`
  2230. // ReportedProgress: The worker's progress through this WorkItem.
  2231. ReportedProgress *ApproximateReportedProgress `json:"reportedProgress,omitempty"`
  2232. // RequestedLeaseDuration: Amount of time the worker requests for its
  2233. // lease.
  2234. RequestedLeaseDuration string `json:"requestedLeaseDuration,omitempty"`
  2235. // SourceFork: DEPRECATED in favor of dynamic_source_split.
  2236. SourceFork *SourceFork `json:"sourceFork,omitempty"`
  2237. // SourceOperationResponse: If the work item represented a
  2238. // SourceOperationRequest, and the work is completed, contains the
  2239. // result of the operation.
  2240. SourceOperationResponse *SourceOperationResponse `json:"sourceOperationResponse,omitempty"`
  2241. // StopPosition: A worker may split an active map task in two parts,
  2242. // "primary" and "residual", continuing to process the primary part and
  2243. // returning the residual part into the pool of available work. This
  2244. // event is called a "dynamic split" and is critical to the dynamic work
  2245. // rebalancing feature. The two obtained sub-tasks are called "parts" of
  2246. // the split. The parts, if concatenated, must represent the same input
  2247. // as would be read by the current task if the split did not happen. The
  2248. // exact way in which the original task is decomposed into the two parts
  2249. // is specified either as a position demarcating them (stop_position),
  2250. // or explicitly as two DerivedSources, if this task consumes a
  2251. // user-defined source type (dynamic_source_split). The "current" task
  2252. // is adjusted as a result of the split: after a task with range [A, B)
  2253. // sends a stop_position update at C, its range is considered to be [A,
  2254. // C), e.g.: * Progress should be interpreted relative to the new range,
  2255. // e.g. "75% completed" means "75% of [A, C) completed" * The worker
  2256. // should interpret proposed_stop_position relative to the new range,
  2257. // e.g. "split at 68%" should be interpreted as "split at 68% of [A,
  2258. // C)". * If the worker chooses to split again using stop_position, only
  2259. // stop_positions in [A, C) will be accepted. * Etc.
  2260. // dynamic_source_split has similar semantics: e.g., if a task with
  2261. // source S splits using dynamic_source_split into {P, R} (where P and R
  2262. // must be together equivalent to S), then subsequent progress and
  2263. // proposed_stop_position should be interpreted relative to P, and in a
  2264. // potential subsequent dynamic_source_split into {P', R'}, P' and R'
  2265. // must be together equivalent to P, etc.
  2266. StopPosition *Position `json:"stopPosition,omitempty"`
  2267. // WorkItemId: Identifies the WorkItem.
  2268. WorkItemId string `json:"workItemId,omitempty"`
  2269. // ForceSendFields is a list of field names (e.g. "Completed") to
  2270. // unconditionally include in API requests. By default, fields with
  2271. // empty values are omitted from API requests. However, any non-pointer,
  2272. // non-interface field appearing in ForceSendFields will be sent to the
  2273. // server regardless of whether the field is empty or not. This may be
  2274. // used to include empty fields in Patch requests.
  2275. ForceSendFields []string `json:"-"`
  2276. }
  2277. func (s *WorkItemStatus) MarshalJSON() ([]byte, error) {
  2278. type noMethod WorkItemStatus
  2279. raw := noMethod(*s)
  2280. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2281. }
  2282. // WorkerHealthReport: WorkerHealthReport contains information about the
  2283. // health of a worker. The VM should be identified by the labels
  2284. // attached to the WorkerMessage that this health ping belongs to.
  2285. type WorkerHealthReport struct {
  2286. // Pods: The pods running on the worker. See:
  2287. // http://kubernetes.io/v1.1/docs/api-reference/v1/definitions.html#_v1_pod This field is used by the worker to send the status of the indvidual containers running on each
  2288. // worker.
  2289. Pods []WorkerHealthReportPods `json:"pods,omitempty"`
  2290. // ReportInterval: The interval at which the worker is sending health
  2291. // reports. The default value of 0 should be interpreted as the field is
  2292. // not being explicitly set by the worker.
  2293. ReportInterval string `json:"reportInterval,omitempty"`
  2294. // VmIsHealthy: Whether the VM is healthy.
  2295. VmIsHealthy bool `json:"vmIsHealthy,omitempty"`
  2296. // VmStartupTime: The time the VM was booted.
  2297. VmStartupTime string `json:"vmStartupTime,omitempty"`
  2298. // ForceSendFields is a list of field names (e.g. "Pods") to
  2299. // unconditionally include in API requests. By default, fields with
  2300. // empty values are omitted from API requests. However, any non-pointer,
  2301. // non-interface field appearing in ForceSendFields will be sent to the
  2302. // server regardless of whether the field is empty or not. This may be
  2303. // used to include empty fields in Patch requests.
  2304. ForceSendFields []string `json:"-"`
  2305. }
  2306. func (s *WorkerHealthReport) MarshalJSON() ([]byte, error) {
  2307. type noMethod WorkerHealthReport
  2308. raw := noMethod(*s)
  2309. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2310. }
  2311. type WorkerHealthReportPods interface{}
  2312. // WorkerHealthReportResponse: WorkerHealthReportResponse contains
  2313. // information returned to the worker in response to a health ping.
  2314. type WorkerHealthReportResponse struct {
  2315. // ReportInterval: A positive value indicates the worker should change
  2316. // its reporting interval to the specified value. The default value of
  2317. // zero means no change in report rate is requested by the server.
  2318. ReportInterval string `json:"reportInterval,omitempty"`
  2319. // ForceSendFields is a list of field names (e.g. "ReportInterval") to
  2320. // unconditionally include in API requests. By default, fields with
  2321. // empty values are omitted from API requests. However, any non-pointer,
  2322. // non-interface field appearing in ForceSendFields will be sent to the
  2323. // server regardless of whether the field is empty or not. This may be
  2324. // used to include empty fields in Patch requests.
  2325. ForceSendFields []string `json:"-"`
  2326. }
  2327. func (s *WorkerHealthReportResponse) MarshalJSON() ([]byte, error) {
  2328. type noMethod WorkerHealthReportResponse
  2329. raw := noMethod(*s)
  2330. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2331. }
  2332. // WorkerMessage: WorkerMessage provides information to the backend
  2333. // about a worker.
  2334. type WorkerMessage struct {
  2335. // Labels: Labels are used to group WorkerMessages. For example, a
  2336. // worker_message about a particular container might have the labels: {
  2337. // "JOB_ID": "2015-04-22", "WORKER_ID": "wordcount-vm-2015…"
  2338. // "CONTAINER_TYPE": "worker", "CONTAINER_ID": "ac1234def"} Label tags
  2339. // typically correspond to Label enum values. However, for ease of
  2340. // development other strings can be used as tags. LABEL_UNSPECIFIED
  2341. // should not be used here.
  2342. Labels map[string]string `json:"labels,omitempty"`
  2343. // Time: The timestamp of the worker_message.
  2344. Time string `json:"time,omitempty"`
  2345. // WorkerHealthReport: The health of a worker.
  2346. WorkerHealthReport *WorkerHealthReport `json:"workerHealthReport,omitempty"`
  2347. // WorkerMessageCode: A worker message code.
  2348. WorkerMessageCode *WorkerMessageCode `json:"workerMessageCode,omitempty"`
  2349. // ForceSendFields is a list of field names (e.g. "Labels") to
  2350. // unconditionally include in API requests. By default, fields with
  2351. // empty values are omitted from API requests. However, any non-pointer,
  2352. // non-interface field appearing in ForceSendFields will be sent to the
  2353. // server regardless of whether the field is empty or not. This may be
  2354. // used to include empty fields in Patch requests.
  2355. ForceSendFields []string `json:"-"`
  2356. }
  2357. func (s *WorkerMessage) MarshalJSON() ([]byte, error) {
  2358. type noMethod WorkerMessage
  2359. raw := noMethod(*s)
  2360. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2361. }
  2362. // WorkerMessageCode: A message code is used to report status and error
  2363. // messages to the service. The message codes are intended to be machine
  2364. // readable. The service will take care of translating these into user
  2365. // understandable messages if necessary. Example use cases: 1. Worker
  2366. // processes reporting successful startup. 2. Worker processes reporting
  2367. // specific errors (e.g. package staging failure).
  2368. type WorkerMessageCode struct {
  2369. // Code: The code is a string intended for consumption by a machine that
  2370. // identifies the type of message being sent. Examples: 1.
  2371. // "HARNESS_STARTED" might be used to indicate the worker harness has
  2372. // started. 2. "GCS_DOWNLOAD_ERROR" might be used to indicate an error
  2373. // downloading a GCS file as part of the boot process of one of the
  2374. // worker containers. This is a string and not an enum to make it easy
  2375. // to add new codes without waiting for an API change.
  2376. Code string `json:"code,omitempty"`
  2377. // Parameters: Parameters contains specific information about the code.
  2378. // This is a struct to allow parameters of different types. Examples: 1.
  2379. // For a "HARNESS_STARTED" message parameters might provide the name of
  2380. // the worker and additional data like timing information. 2. For a
  2381. // "GCS_DOWNLOAD_ERROR" parameters might contain fields listing the GCS
  2382. // objects being downloaded and fields containing errors. In general
  2383. // complex data structures should be avoided. If a worker needs to send
  2384. // a specific and complicated data structure then please consider
  2385. // defining a new proto and adding it to the data oneof in
  2386. // WorkerMessageResponse. Conventions: Parameters should only be used
  2387. // for information that isn't typically passed as a label. hostname and
  2388. // other worker identifiers should almost always be passed as labels
  2389. // since they will be included on most messages.
  2390. Parameters WorkerMessageCodeParameters `json:"parameters,omitempty"`
  2391. // ForceSendFields is a list of field names (e.g. "Code") to
  2392. // unconditionally include in API requests. By default, fields with
  2393. // empty values are omitted from API requests. However, any non-pointer,
  2394. // non-interface field appearing in ForceSendFields will be sent to the
  2395. // server regardless of whether the field is empty or not. This may be
  2396. // used to include empty fields in Patch requests.
  2397. ForceSendFields []string `json:"-"`
  2398. }
  2399. func (s *WorkerMessageCode) MarshalJSON() ([]byte, error) {
  2400. type noMethod WorkerMessageCode
  2401. raw := noMethod(*s)
  2402. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2403. }
  2404. type WorkerMessageCodeParameters interface{}
  2405. // WorkerMessageResponse: A worker_message response allows the server to
  2406. // pass information to the sender.
  2407. type WorkerMessageResponse struct {
  2408. // WorkerHealthReportResponse: The service's response to a worker's
  2409. // health report.
  2410. WorkerHealthReportResponse *WorkerHealthReportResponse `json:"workerHealthReportResponse,omitempty"`
  2411. // ForceSendFields is a list of field names (e.g.
  2412. // "WorkerHealthReportResponse") to unconditionally include in API
  2413. // requests. By default, fields with empty values are omitted from API
  2414. // requests. However, any non-pointer, non-interface field appearing in
  2415. // ForceSendFields will be sent to the server regardless of whether the
  2416. // field is empty or not. This may be used to include empty fields in
  2417. // Patch requests.
  2418. ForceSendFields []string `json:"-"`
  2419. }
  2420. func (s *WorkerMessageResponse) MarshalJSON() ([]byte, error) {
  2421. type noMethod WorkerMessageResponse
  2422. raw := noMethod(*s)
  2423. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2424. }
  2425. // WorkerPool: Describes one particular pool of Dataflow workers to be
  2426. // instantiated by the Dataflow service in order to perform the
  2427. // computations required by a job. Note that a workflow job may use
  2428. // multiple pools, in order to match the various computational
  2429. // requirements of the various stages of the job.
  2430. type WorkerPool struct {
  2431. // AutoscalingSettings: Settings for autoscaling of this WorkerPool.
  2432. AutoscalingSettings *AutoscalingSettings `json:"autoscalingSettings,omitempty"`
  2433. // DataDisks: Data disks that are used by a VM in this workflow.
  2434. DataDisks []*Disk `json:"dataDisks,omitempty"`
  2435. // DefaultPackageSet: The default package set to install. This allows
  2436. // the service to select a default set of packages which are useful to
  2437. // worker harnesses written in a particular language.
  2438. //
  2439. // Possible values:
  2440. // "DEFAULT_PACKAGE_SET_UNKNOWN"
  2441. // "DEFAULT_PACKAGE_SET_NONE"
  2442. // "DEFAULT_PACKAGE_SET_JAVA"
  2443. // "DEFAULT_PACKAGE_SET_PYTHON"
  2444. DefaultPackageSet string `json:"defaultPackageSet,omitempty"`
  2445. // DiskSizeGb: Size of root disk for VMs, in GB. If zero or unspecified,
  2446. // the service will attempt to choose a reasonable default.
  2447. DiskSizeGb int64 `json:"diskSizeGb,omitempty"`
  2448. // DiskSourceImage: Fully qualified source image for disks.
  2449. DiskSourceImage string `json:"diskSourceImage,omitempty"`
  2450. // DiskType: Type of root disk for VMs. If empty or unspecified, the
  2451. // service will attempt to choose a reasonable default.
  2452. DiskType string `json:"diskType,omitempty"`
  2453. // Kind: The kind of the worker pool; currently only 'harness' and
  2454. // 'shuffle' are supported.
  2455. Kind string `json:"kind,omitempty"`
  2456. // MachineType: Machine type (e.g. "n1-standard-1"). If empty or
  2457. // unspecified, the service will attempt to choose a reasonable default.
  2458. MachineType string `json:"machineType,omitempty"`
  2459. // Metadata: Metadata to set on the Google Compute Engine VMs.
  2460. Metadata map[string]string `json:"metadata,omitempty"`
  2461. // Network: Network to which VMs will be assigned. If empty or
  2462. // unspecified, the service will use the network "default".
  2463. Network string `json:"network,omitempty"`
  2464. // NumWorkers: Number of Google Compute Engine workers in this pool
  2465. // needed to execute the job. If zero or unspecified, the service will
  2466. // attempt to choose a reasonable default.
  2467. NumWorkers int64 `json:"numWorkers,omitempty"`
  2468. // OnHostMaintenance: The action to take on host maintenance, as defined
  2469. // by the Google Compute Engine API.
  2470. OnHostMaintenance string `json:"onHostMaintenance,omitempty"`
  2471. // Packages: Packages to be installed on workers.
  2472. Packages []*Package `json:"packages,omitempty"`
  2473. // PoolArgs: Extra arguments for this worker pool.
  2474. PoolArgs WorkerPoolPoolArgs `json:"poolArgs,omitempty"`
  2475. // TaskrunnerSettings: Settings passed through to Google Compute Engine
  2476. // workers when using the standard Dataflow task runner. Users should
  2477. // ignore this field.
  2478. TaskrunnerSettings *TaskRunnerSettings `json:"taskrunnerSettings,omitempty"`
  2479. // TeardownPolicy: Sets the policy for determining when to turndown
  2480. // worker pool. Allowed values are: TEARDOWN_ALWAYS,
  2481. // TEARDOWN_ON_SUCCESS, and TEARDOWN_NEVER. TEARDOWN_ALWAYS means
  2482. // workers are always torn down regardless of whether the job succeeds.
  2483. // TEARDOWN_ON_SUCCESS means workers are torn down if the job succeeds.
  2484. // TEARDOWN_NEVER means the workers are never torn down. If the workers
  2485. // are not torn down by the service, they will continue to run and use
  2486. // Google Compute Engine VM resources in the user's project until they
  2487. // are explicitly terminated by the user. Because of this, Google
  2488. // recommends using the TEARDOWN_ALWAYS policy except for small,
  2489. // manually supervised test jobs. If unknown or unspecified, the service
  2490. // will attempt to choose a reasonable default.
  2491. //
  2492. // Possible values:
  2493. // "TEARDOWN_POLICY_UNKNOWN"
  2494. // "TEARDOWN_ALWAYS"
  2495. // "TEARDOWN_ON_SUCCESS"
  2496. // "TEARDOWN_NEVER"
  2497. TeardownPolicy string `json:"teardownPolicy,omitempty"`
  2498. // Zone: Zone to run the worker pools in. If empty or unspecified, the
  2499. // service will attempt to choose a reasonable default.
  2500. Zone string `json:"zone,omitempty"`
  2501. // ForceSendFields is a list of field names (e.g. "AutoscalingSettings")
  2502. // to unconditionally include in API requests. By default, fields with
  2503. // empty values are omitted from API requests. However, any non-pointer,
  2504. // non-interface field appearing in ForceSendFields will be sent to the
  2505. // server regardless of whether the field is empty or not. This may be
  2506. // used to include empty fields in Patch requests.
  2507. ForceSendFields []string `json:"-"`
  2508. }
  2509. func (s *WorkerPool) MarshalJSON() ([]byte, error) {
  2510. type noMethod WorkerPool
  2511. raw := noMethod(*s)
  2512. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2513. }
  2514. type WorkerPoolPoolArgs interface{}
  2515. // WorkerSettings: Provides data to pass through to the worker harness.
  2516. type WorkerSettings struct {
  2517. // BaseUrl: The base URL for accessing Google Cloud APIs. When workers
  2518. // access Google Cloud APIs, they logically do so via relative URLs. If
  2519. // this field is specified, it supplies the base URL to use for
  2520. // resolving these relative URLs. The normative algorithm used is
  2521. // defined by RFC 1808, "Relative Uniform Resource Locators". If not
  2522. // specified, the default value is "http://www.googleapis.com/"
  2523. BaseUrl string `json:"baseUrl,omitempty"`
  2524. // ReportingEnabled: Send work progress updates to service.
  2525. ReportingEnabled bool `json:"reportingEnabled,omitempty"`
  2526. // ServicePath: The Dataflow service path relative to the root URL, for
  2527. // example, "dataflow/v1b3/projects".
  2528. ServicePath string `json:"servicePath,omitempty"`
  2529. // ShuffleServicePath: The Shuffle service path relative to the root
  2530. // URL, for example, "shuffle/v1beta1".
  2531. ShuffleServicePath string `json:"shuffleServicePath,omitempty"`
  2532. // TempStoragePrefix: The prefix of the resources the system should use
  2533. // for temporary storage. The supported resource type is: Google Cloud
  2534. // Storage: storage.googleapis.com/{bucket}/{object}
  2535. // bucket.storage.googleapis.com/{object}
  2536. TempStoragePrefix string `json:"tempStoragePrefix,omitempty"`
  2537. // WorkerId: ID of the worker running this pipeline.
  2538. WorkerId string `json:"workerId,omitempty"`
  2539. // ForceSendFields is a list of field names (e.g. "BaseUrl") to
  2540. // unconditionally include in API requests. By default, fields with
  2541. // empty values are omitted from API requests. However, any non-pointer,
  2542. // non-interface field appearing in ForceSendFields will be sent to the
  2543. // server regardless of whether the field is empty or not. This may be
  2544. // used to include empty fields in Patch requests.
  2545. ForceSendFields []string `json:"-"`
  2546. }
  2547. func (s *WorkerSettings) MarshalJSON() ([]byte, error) {
  2548. type noMethod WorkerSettings
  2549. raw := noMethod(*s)
  2550. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2551. }
  2552. // WriteInstruction: An instruction that writes records. Takes one
  2553. // input, produces no outputs.
  2554. type WriteInstruction struct {
  2555. // Input: The input.
  2556. Input *InstructionInput `json:"input,omitempty"`
  2557. // Sink: The sink to write to.
  2558. Sink *Sink `json:"sink,omitempty"`
  2559. // ForceSendFields is a list of field names (e.g. "Input") to
  2560. // unconditionally include in API requests. By default, fields with
  2561. // empty values are omitted from API requests. However, any non-pointer,
  2562. // non-interface field appearing in ForceSendFields will be sent to the
  2563. // server regardless of whether the field is empty or not. This may be
  2564. // used to include empty fields in Patch requests.
  2565. ForceSendFields []string `json:"-"`
  2566. }
  2567. func (s *WriteInstruction) MarshalJSON() ([]byte, error) {
  2568. type noMethod WriteInstruction
  2569. raw := noMethod(*s)
  2570. return gensupport.MarshalJSON(raw, s.ForceSendFields)
  2571. }
  2572. // method id "dataflow.projects.workerMessages":
  2573. type ProjectsWorkerMessagesCall struct {
  2574. s *Service
  2575. projectId string
  2576. sendworkermessagesrequest *SendWorkerMessagesRequest
  2577. urlParams_ gensupport.URLParams
  2578. ctx_ context.Context
  2579. }
  2580. // WorkerMessages: Send a worker_message to the service.
  2581. func (r *ProjectsService) WorkerMessages(projectId string, sendworkermessagesrequest *SendWorkerMessagesRequest) *ProjectsWorkerMessagesCall {
  2582. c := &ProjectsWorkerMessagesCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  2583. c.projectId = projectId
  2584. c.sendworkermessagesrequest = sendworkermessagesrequest
  2585. return c
  2586. }
  2587. // Fields allows partial responses to be retrieved. See
  2588. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  2589. // for more information.
  2590. func (c *ProjectsWorkerMessagesCall) Fields(s ...googleapi.Field) *ProjectsWorkerMessagesCall {
  2591. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  2592. return c
  2593. }
  2594. // Context sets the context to be used in this call's Do method. Any
  2595. // pending HTTP request will be aborted if the provided context is
  2596. // canceled.
  2597. func (c *ProjectsWorkerMessagesCall) Context(ctx context.Context) *ProjectsWorkerMessagesCall {
  2598. c.ctx_ = ctx
  2599. return c
  2600. }
  2601. func (c *ProjectsWorkerMessagesCall) doRequest(alt string) (*http.Response, error) {
  2602. var body io.Reader = nil
  2603. body, err := googleapi.WithoutDataWrapper.JSONReader(c.sendworkermessagesrequest)
  2604. if err != nil {
  2605. return nil, err
  2606. }
  2607. ctype := "application/json"
  2608. c.urlParams_.Set("alt", alt)
  2609. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/WorkerMessages")
  2610. urls += "?" + c.urlParams_.Encode()
  2611. req, _ := http.NewRequest("POST", urls, body)
  2612. googleapi.Expand(req.URL, map[string]string{
  2613. "projectId": c.projectId,
  2614. })
  2615. req.Header.Set("Content-Type", ctype)
  2616. req.Header.Set("User-Agent", c.s.userAgent())
  2617. if c.ctx_ != nil {
  2618. return ctxhttp.Do(c.ctx_, c.s.client, req)
  2619. }
  2620. return c.s.client.Do(req)
  2621. }
  2622. // Do executes the "dataflow.projects.workerMessages" call.
  2623. // Exactly one of *SendWorkerMessagesResponse or error will be non-nil.
  2624. // Any non-2xx status code is an error. Response headers are in either
  2625. // *SendWorkerMessagesResponse.ServerResponse.Header or (if a response
  2626. // was returned at all) in error.(*googleapi.Error).Header. Use
  2627. // googleapi.IsNotModified to check whether the returned error was
  2628. // because http.StatusNotModified was returned.
  2629. func (c *ProjectsWorkerMessagesCall) Do(opts ...googleapi.CallOption) (*SendWorkerMessagesResponse, error) {
  2630. gensupport.SetOptions(c.urlParams_, opts...)
  2631. res, err := c.doRequest("json")
  2632. if res != nil && res.StatusCode == http.StatusNotModified {
  2633. if res.Body != nil {
  2634. res.Body.Close()
  2635. }
  2636. return nil, &googleapi.Error{
  2637. Code: res.StatusCode,
  2638. Header: res.Header,
  2639. }
  2640. }
  2641. if err != nil {
  2642. return nil, err
  2643. }
  2644. defer googleapi.CloseBody(res)
  2645. if err := googleapi.CheckResponse(res); err != nil {
  2646. return nil, err
  2647. }
  2648. ret := &SendWorkerMessagesResponse{
  2649. ServerResponse: googleapi.ServerResponse{
  2650. Header: res.Header,
  2651. HTTPStatusCode: res.StatusCode,
  2652. },
  2653. }
  2654. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  2655. return nil, err
  2656. }
  2657. return ret, nil
  2658. // {
  2659. // "description": "Send a worker_message to the service.",
  2660. // "httpMethod": "POST",
  2661. // "id": "dataflow.projects.workerMessages",
  2662. // "parameterOrder": [
  2663. // "projectId"
  2664. // ],
  2665. // "parameters": {
  2666. // "projectId": {
  2667. // "description": "The project to send the WorkerMessages to.",
  2668. // "location": "path",
  2669. // "required": true,
  2670. // "type": "string"
  2671. // }
  2672. // },
  2673. // "path": "v1b3/projects/{projectId}/WorkerMessages",
  2674. // "request": {
  2675. // "$ref": "SendWorkerMessagesRequest"
  2676. // },
  2677. // "response": {
  2678. // "$ref": "SendWorkerMessagesResponse"
  2679. // },
  2680. // "scopes": [
  2681. // "https://www.googleapis.com/auth/cloud-platform",
  2682. // "https://www.googleapis.com/auth/userinfo.email"
  2683. // ]
  2684. // }
  2685. }
  2686. // method id "dataflow.projects.jobs.create":
  2687. type ProjectsJobsCreateCall struct {
  2688. s *Service
  2689. projectId string
  2690. job *Job
  2691. urlParams_ gensupport.URLParams
  2692. ctx_ context.Context
  2693. }
  2694. // Create: Creates a dataflow job.
  2695. func (r *ProjectsJobsService) Create(projectId string, job *Job) *ProjectsJobsCreateCall {
  2696. c := &ProjectsJobsCreateCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  2697. c.projectId = projectId
  2698. c.job = job
  2699. return c
  2700. }
  2701. // ReplaceJobId sets the optional parameter "replaceJobId": DEPRECATED.
  2702. // This field is now on the Job message.
  2703. func (c *ProjectsJobsCreateCall) ReplaceJobId(replaceJobId string) *ProjectsJobsCreateCall {
  2704. c.urlParams_.Set("replaceJobId", replaceJobId)
  2705. return c
  2706. }
  2707. // View sets the optional parameter "view": Level of information
  2708. // requested in response.
  2709. //
  2710. // Possible values:
  2711. // "JOB_VIEW_UNKNOWN"
  2712. // "JOB_VIEW_SUMMARY"
  2713. // "JOB_VIEW_ALL"
  2714. func (c *ProjectsJobsCreateCall) View(view string) *ProjectsJobsCreateCall {
  2715. c.urlParams_.Set("view", view)
  2716. return c
  2717. }
  2718. // Fields allows partial responses to be retrieved. See
  2719. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  2720. // for more information.
  2721. func (c *ProjectsJobsCreateCall) Fields(s ...googleapi.Field) *ProjectsJobsCreateCall {
  2722. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  2723. return c
  2724. }
  2725. // Context sets the context to be used in this call's Do method. Any
  2726. // pending HTTP request will be aborted if the provided context is
  2727. // canceled.
  2728. func (c *ProjectsJobsCreateCall) Context(ctx context.Context) *ProjectsJobsCreateCall {
  2729. c.ctx_ = ctx
  2730. return c
  2731. }
  2732. func (c *ProjectsJobsCreateCall) doRequest(alt string) (*http.Response, error) {
  2733. var body io.Reader = nil
  2734. body, err := googleapi.WithoutDataWrapper.JSONReader(c.job)
  2735. if err != nil {
  2736. return nil, err
  2737. }
  2738. ctype := "application/json"
  2739. c.urlParams_.Set("alt", alt)
  2740. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/jobs")
  2741. urls += "?" + c.urlParams_.Encode()
  2742. req, _ := http.NewRequest("POST", urls, body)
  2743. googleapi.Expand(req.URL, map[string]string{
  2744. "projectId": c.projectId,
  2745. })
  2746. req.Header.Set("Content-Type", ctype)
  2747. req.Header.Set("User-Agent", c.s.userAgent())
  2748. if c.ctx_ != nil {
  2749. return ctxhttp.Do(c.ctx_, c.s.client, req)
  2750. }
  2751. return c.s.client.Do(req)
  2752. }
  2753. // Do executes the "dataflow.projects.jobs.create" call.
  2754. // Exactly one of *Job or error will be non-nil. Any non-2xx status code
  2755. // is an error. Response headers are in either
  2756. // *Job.ServerResponse.Header or (if a response was returned at all) in
  2757. // error.(*googleapi.Error).Header. Use googleapi.IsNotModified to check
  2758. // whether the returned error was because http.StatusNotModified was
  2759. // returned.
  2760. func (c *ProjectsJobsCreateCall) Do(opts ...googleapi.CallOption) (*Job, error) {
  2761. gensupport.SetOptions(c.urlParams_, opts...)
  2762. res, err := c.doRequest("json")
  2763. if res != nil && res.StatusCode == http.StatusNotModified {
  2764. if res.Body != nil {
  2765. res.Body.Close()
  2766. }
  2767. return nil, &googleapi.Error{
  2768. Code: res.StatusCode,
  2769. Header: res.Header,
  2770. }
  2771. }
  2772. if err != nil {
  2773. return nil, err
  2774. }
  2775. defer googleapi.CloseBody(res)
  2776. if err := googleapi.CheckResponse(res); err != nil {
  2777. return nil, err
  2778. }
  2779. ret := &Job{
  2780. ServerResponse: googleapi.ServerResponse{
  2781. Header: res.Header,
  2782. HTTPStatusCode: res.StatusCode,
  2783. },
  2784. }
  2785. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  2786. return nil, err
  2787. }
  2788. return ret, nil
  2789. // {
  2790. // "description": "Creates a dataflow job.",
  2791. // "httpMethod": "POST",
  2792. // "id": "dataflow.projects.jobs.create",
  2793. // "parameterOrder": [
  2794. // "projectId"
  2795. // ],
  2796. // "parameters": {
  2797. // "projectId": {
  2798. // "description": "The project which owns the job.",
  2799. // "location": "path",
  2800. // "required": true,
  2801. // "type": "string"
  2802. // },
  2803. // "replaceJobId": {
  2804. // "description": "DEPRECATED. This field is now on the Job message.",
  2805. // "location": "query",
  2806. // "type": "string"
  2807. // },
  2808. // "view": {
  2809. // "description": "Level of information requested in response.",
  2810. // "enum": [
  2811. // "JOB_VIEW_UNKNOWN",
  2812. // "JOB_VIEW_SUMMARY",
  2813. // "JOB_VIEW_ALL"
  2814. // ],
  2815. // "location": "query",
  2816. // "type": "string"
  2817. // }
  2818. // },
  2819. // "path": "v1b3/projects/{projectId}/jobs",
  2820. // "request": {
  2821. // "$ref": "Job"
  2822. // },
  2823. // "response": {
  2824. // "$ref": "Job"
  2825. // },
  2826. // "scopes": [
  2827. // "https://www.googleapis.com/auth/cloud-platform",
  2828. // "https://www.googleapis.com/auth/userinfo.email"
  2829. // ]
  2830. // }
  2831. }
  2832. // method id "dataflow.projects.jobs.get":
  2833. type ProjectsJobsGetCall struct {
  2834. s *Service
  2835. projectId string
  2836. jobId string
  2837. urlParams_ gensupport.URLParams
  2838. ifNoneMatch_ string
  2839. ctx_ context.Context
  2840. }
  2841. // Get: Gets the state of the specified dataflow job.
  2842. func (r *ProjectsJobsService) Get(projectId string, jobId string) *ProjectsJobsGetCall {
  2843. c := &ProjectsJobsGetCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  2844. c.projectId = projectId
  2845. c.jobId = jobId
  2846. return c
  2847. }
  2848. // View sets the optional parameter "view": Level of information
  2849. // requested in response.
  2850. //
  2851. // Possible values:
  2852. // "JOB_VIEW_UNKNOWN"
  2853. // "JOB_VIEW_SUMMARY"
  2854. // "JOB_VIEW_ALL"
  2855. func (c *ProjectsJobsGetCall) View(view string) *ProjectsJobsGetCall {
  2856. c.urlParams_.Set("view", view)
  2857. return c
  2858. }
  2859. // Fields allows partial responses to be retrieved. See
  2860. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  2861. // for more information.
  2862. func (c *ProjectsJobsGetCall) Fields(s ...googleapi.Field) *ProjectsJobsGetCall {
  2863. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  2864. return c
  2865. }
  2866. // IfNoneMatch sets the optional parameter which makes the operation
  2867. // fail if the object's ETag matches the given value. This is useful for
  2868. // getting updates only after the object has changed since the last
  2869. // request. Use googleapi.IsNotModified to check whether the response
  2870. // error from Do is the result of In-None-Match.
  2871. func (c *ProjectsJobsGetCall) IfNoneMatch(entityTag string) *ProjectsJobsGetCall {
  2872. c.ifNoneMatch_ = entityTag
  2873. return c
  2874. }
  2875. // Context sets the context to be used in this call's Do method. Any
  2876. // pending HTTP request will be aborted if the provided context is
  2877. // canceled.
  2878. func (c *ProjectsJobsGetCall) Context(ctx context.Context) *ProjectsJobsGetCall {
  2879. c.ctx_ = ctx
  2880. return c
  2881. }
  2882. func (c *ProjectsJobsGetCall) doRequest(alt string) (*http.Response, error) {
  2883. var body io.Reader = nil
  2884. c.urlParams_.Set("alt", alt)
  2885. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/jobs/{jobId}")
  2886. urls += "?" + c.urlParams_.Encode()
  2887. req, _ := http.NewRequest("GET", urls, body)
  2888. googleapi.Expand(req.URL, map[string]string{
  2889. "projectId": c.projectId,
  2890. "jobId": c.jobId,
  2891. })
  2892. req.Header.Set("User-Agent", c.s.userAgent())
  2893. if c.ifNoneMatch_ != "" {
  2894. req.Header.Set("If-None-Match", c.ifNoneMatch_)
  2895. }
  2896. if c.ctx_ != nil {
  2897. return ctxhttp.Do(c.ctx_, c.s.client, req)
  2898. }
  2899. return c.s.client.Do(req)
  2900. }
  2901. // Do executes the "dataflow.projects.jobs.get" call.
  2902. // Exactly one of *Job or error will be non-nil. Any non-2xx status code
  2903. // is an error. Response headers are in either
  2904. // *Job.ServerResponse.Header or (if a response was returned at all) in
  2905. // error.(*googleapi.Error).Header. Use googleapi.IsNotModified to check
  2906. // whether the returned error was because http.StatusNotModified was
  2907. // returned.
  2908. func (c *ProjectsJobsGetCall) Do(opts ...googleapi.CallOption) (*Job, error) {
  2909. gensupport.SetOptions(c.urlParams_, opts...)
  2910. res, err := c.doRequest("json")
  2911. if res != nil && res.StatusCode == http.StatusNotModified {
  2912. if res.Body != nil {
  2913. res.Body.Close()
  2914. }
  2915. return nil, &googleapi.Error{
  2916. Code: res.StatusCode,
  2917. Header: res.Header,
  2918. }
  2919. }
  2920. if err != nil {
  2921. return nil, err
  2922. }
  2923. defer googleapi.CloseBody(res)
  2924. if err := googleapi.CheckResponse(res); err != nil {
  2925. return nil, err
  2926. }
  2927. ret := &Job{
  2928. ServerResponse: googleapi.ServerResponse{
  2929. Header: res.Header,
  2930. HTTPStatusCode: res.StatusCode,
  2931. },
  2932. }
  2933. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  2934. return nil, err
  2935. }
  2936. return ret, nil
  2937. // {
  2938. // "description": "Gets the state of the specified dataflow job.",
  2939. // "httpMethod": "GET",
  2940. // "id": "dataflow.projects.jobs.get",
  2941. // "parameterOrder": [
  2942. // "projectId",
  2943. // "jobId"
  2944. // ],
  2945. // "parameters": {
  2946. // "jobId": {
  2947. // "description": "Identifies a single job.",
  2948. // "location": "path",
  2949. // "required": true,
  2950. // "type": "string"
  2951. // },
  2952. // "projectId": {
  2953. // "description": "The project which owns the job.",
  2954. // "location": "path",
  2955. // "required": true,
  2956. // "type": "string"
  2957. // },
  2958. // "view": {
  2959. // "description": "Level of information requested in response.",
  2960. // "enum": [
  2961. // "JOB_VIEW_UNKNOWN",
  2962. // "JOB_VIEW_SUMMARY",
  2963. // "JOB_VIEW_ALL"
  2964. // ],
  2965. // "location": "query",
  2966. // "type": "string"
  2967. // }
  2968. // },
  2969. // "path": "v1b3/projects/{projectId}/jobs/{jobId}",
  2970. // "response": {
  2971. // "$ref": "Job"
  2972. // },
  2973. // "scopes": [
  2974. // "https://www.googleapis.com/auth/cloud-platform",
  2975. // "https://www.googleapis.com/auth/userinfo.email"
  2976. // ]
  2977. // }
  2978. }
  2979. // method id "dataflow.projects.jobs.getMetrics":
  2980. type ProjectsJobsGetMetricsCall struct {
  2981. s *Service
  2982. projectId string
  2983. jobId string
  2984. urlParams_ gensupport.URLParams
  2985. ifNoneMatch_ string
  2986. ctx_ context.Context
  2987. }
  2988. // GetMetrics: Request the job status.
  2989. func (r *ProjectsJobsService) GetMetrics(projectId string, jobId string) *ProjectsJobsGetMetricsCall {
  2990. c := &ProjectsJobsGetMetricsCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  2991. c.projectId = projectId
  2992. c.jobId = jobId
  2993. return c
  2994. }
  2995. // StartTime sets the optional parameter "startTime": Return only metric
  2996. // data that has changed since this time. Default is to return all
  2997. // information about all metrics for the job.
  2998. func (c *ProjectsJobsGetMetricsCall) StartTime(startTime string) *ProjectsJobsGetMetricsCall {
  2999. c.urlParams_.Set("startTime", startTime)
  3000. return c
  3001. }
  3002. // Fields allows partial responses to be retrieved. See
  3003. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  3004. // for more information.
  3005. func (c *ProjectsJobsGetMetricsCall) Fields(s ...googleapi.Field) *ProjectsJobsGetMetricsCall {
  3006. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  3007. return c
  3008. }
  3009. // IfNoneMatch sets the optional parameter which makes the operation
  3010. // fail if the object's ETag matches the given value. This is useful for
  3011. // getting updates only after the object has changed since the last
  3012. // request. Use googleapi.IsNotModified to check whether the response
  3013. // error from Do is the result of In-None-Match.
  3014. func (c *ProjectsJobsGetMetricsCall) IfNoneMatch(entityTag string) *ProjectsJobsGetMetricsCall {
  3015. c.ifNoneMatch_ = entityTag
  3016. return c
  3017. }
  3018. // Context sets the context to be used in this call's Do method. Any
  3019. // pending HTTP request will be aborted if the provided context is
  3020. // canceled.
  3021. func (c *ProjectsJobsGetMetricsCall) Context(ctx context.Context) *ProjectsJobsGetMetricsCall {
  3022. c.ctx_ = ctx
  3023. return c
  3024. }
  3025. func (c *ProjectsJobsGetMetricsCall) doRequest(alt string) (*http.Response, error) {
  3026. var body io.Reader = nil
  3027. c.urlParams_.Set("alt", alt)
  3028. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/jobs/{jobId}/metrics")
  3029. urls += "?" + c.urlParams_.Encode()
  3030. req, _ := http.NewRequest("GET", urls, body)
  3031. googleapi.Expand(req.URL, map[string]string{
  3032. "projectId": c.projectId,
  3033. "jobId": c.jobId,
  3034. })
  3035. req.Header.Set("User-Agent", c.s.userAgent())
  3036. if c.ifNoneMatch_ != "" {
  3037. req.Header.Set("If-None-Match", c.ifNoneMatch_)
  3038. }
  3039. if c.ctx_ != nil {
  3040. return ctxhttp.Do(c.ctx_, c.s.client, req)
  3041. }
  3042. return c.s.client.Do(req)
  3043. }
  3044. // Do executes the "dataflow.projects.jobs.getMetrics" call.
  3045. // Exactly one of *JobMetrics or error will be non-nil. Any non-2xx
  3046. // status code is an error. Response headers are in either
  3047. // *JobMetrics.ServerResponse.Header or (if a response was returned at
  3048. // all) in error.(*googleapi.Error).Header. Use googleapi.IsNotModified
  3049. // to check whether the returned error was because
  3050. // http.StatusNotModified was returned.
  3051. func (c *ProjectsJobsGetMetricsCall) Do(opts ...googleapi.CallOption) (*JobMetrics, error) {
  3052. gensupport.SetOptions(c.urlParams_, opts...)
  3053. res, err := c.doRequest("json")
  3054. if res != nil && res.StatusCode == http.StatusNotModified {
  3055. if res.Body != nil {
  3056. res.Body.Close()
  3057. }
  3058. return nil, &googleapi.Error{
  3059. Code: res.StatusCode,
  3060. Header: res.Header,
  3061. }
  3062. }
  3063. if err != nil {
  3064. return nil, err
  3065. }
  3066. defer googleapi.CloseBody(res)
  3067. if err := googleapi.CheckResponse(res); err != nil {
  3068. return nil, err
  3069. }
  3070. ret := &JobMetrics{
  3071. ServerResponse: googleapi.ServerResponse{
  3072. Header: res.Header,
  3073. HTTPStatusCode: res.StatusCode,
  3074. },
  3075. }
  3076. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  3077. return nil, err
  3078. }
  3079. return ret, nil
  3080. // {
  3081. // "description": "Request the job status.",
  3082. // "httpMethod": "GET",
  3083. // "id": "dataflow.projects.jobs.getMetrics",
  3084. // "parameterOrder": [
  3085. // "projectId",
  3086. // "jobId"
  3087. // ],
  3088. // "parameters": {
  3089. // "jobId": {
  3090. // "description": "The job to get messages for.",
  3091. // "location": "path",
  3092. // "required": true,
  3093. // "type": "string"
  3094. // },
  3095. // "projectId": {
  3096. // "description": "A project id.",
  3097. // "location": "path",
  3098. // "required": true,
  3099. // "type": "string"
  3100. // },
  3101. // "startTime": {
  3102. // "description": "Return only metric data that has changed since this time. Default is to return all information about all metrics for the job.",
  3103. // "location": "query",
  3104. // "type": "string"
  3105. // }
  3106. // },
  3107. // "path": "v1b3/projects/{projectId}/jobs/{jobId}/metrics",
  3108. // "response": {
  3109. // "$ref": "JobMetrics"
  3110. // },
  3111. // "scopes": [
  3112. // "https://www.googleapis.com/auth/cloud-platform",
  3113. // "https://www.googleapis.com/auth/userinfo.email"
  3114. // ]
  3115. // }
  3116. }
  3117. // method id "dataflow.projects.jobs.list":
  3118. type ProjectsJobsListCall struct {
  3119. s *Service
  3120. projectId string
  3121. urlParams_ gensupport.URLParams
  3122. ifNoneMatch_ string
  3123. ctx_ context.Context
  3124. }
  3125. // List: List the jobs of a project
  3126. func (r *ProjectsJobsService) List(projectId string) *ProjectsJobsListCall {
  3127. c := &ProjectsJobsListCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  3128. c.projectId = projectId
  3129. return c
  3130. }
  3131. // PageSize sets the optional parameter "pageSize": If there are many
  3132. // jobs, limit response to at most this many. The actual number of jobs
  3133. // returned will be the lesser of max_responses and an unspecified
  3134. // server-defined limit.
  3135. func (c *ProjectsJobsListCall) PageSize(pageSize int64) *ProjectsJobsListCall {
  3136. c.urlParams_.Set("pageSize", fmt.Sprint(pageSize))
  3137. return c
  3138. }
  3139. // PageToken sets the optional parameter "pageToken": Set this to the
  3140. // 'next_page_token' field of a previous response to request additional
  3141. // results in a long list.
  3142. func (c *ProjectsJobsListCall) PageToken(pageToken string) *ProjectsJobsListCall {
  3143. c.urlParams_.Set("pageToken", pageToken)
  3144. return c
  3145. }
  3146. // View sets the optional parameter "view": Level of information
  3147. // requested in response. Default is SUMMARY.
  3148. //
  3149. // Possible values:
  3150. // "JOB_VIEW_UNKNOWN"
  3151. // "JOB_VIEW_SUMMARY"
  3152. // "JOB_VIEW_ALL"
  3153. func (c *ProjectsJobsListCall) View(view string) *ProjectsJobsListCall {
  3154. c.urlParams_.Set("view", view)
  3155. return c
  3156. }
  3157. // Fields allows partial responses to be retrieved. See
  3158. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  3159. // for more information.
  3160. func (c *ProjectsJobsListCall) Fields(s ...googleapi.Field) *ProjectsJobsListCall {
  3161. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  3162. return c
  3163. }
  3164. // IfNoneMatch sets the optional parameter which makes the operation
  3165. // fail if the object's ETag matches the given value. This is useful for
  3166. // getting updates only after the object has changed since the last
  3167. // request. Use googleapi.IsNotModified to check whether the response
  3168. // error from Do is the result of In-None-Match.
  3169. func (c *ProjectsJobsListCall) IfNoneMatch(entityTag string) *ProjectsJobsListCall {
  3170. c.ifNoneMatch_ = entityTag
  3171. return c
  3172. }
  3173. // Context sets the context to be used in this call's Do method. Any
  3174. // pending HTTP request will be aborted if the provided context is
  3175. // canceled.
  3176. func (c *ProjectsJobsListCall) Context(ctx context.Context) *ProjectsJobsListCall {
  3177. c.ctx_ = ctx
  3178. return c
  3179. }
  3180. func (c *ProjectsJobsListCall) doRequest(alt string) (*http.Response, error) {
  3181. var body io.Reader = nil
  3182. c.urlParams_.Set("alt", alt)
  3183. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/jobs")
  3184. urls += "?" + c.urlParams_.Encode()
  3185. req, _ := http.NewRequest("GET", urls, body)
  3186. googleapi.Expand(req.URL, map[string]string{
  3187. "projectId": c.projectId,
  3188. })
  3189. req.Header.Set("User-Agent", c.s.userAgent())
  3190. if c.ifNoneMatch_ != "" {
  3191. req.Header.Set("If-None-Match", c.ifNoneMatch_)
  3192. }
  3193. if c.ctx_ != nil {
  3194. return ctxhttp.Do(c.ctx_, c.s.client, req)
  3195. }
  3196. return c.s.client.Do(req)
  3197. }
  3198. // Do executes the "dataflow.projects.jobs.list" call.
  3199. // Exactly one of *ListJobsResponse or error will be non-nil. Any
  3200. // non-2xx status code is an error. Response headers are in either
  3201. // *ListJobsResponse.ServerResponse.Header or (if a response was
  3202. // returned at all) in error.(*googleapi.Error).Header. Use
  3203. // googleapi.IsNotModified to check whether the returned error was
  3204. // because http.StatusNotModified was returned.
  3205. func (c *ProjectsJobsListCall) Do(opts ...googleapi.CallOption) (*ListJobsResponse, error) {
  3206. gensupport.SetOptions(c.urlParams_, opts...)
  3207. res, err := c.doRequest("json")
  3208. if res != nil && res.StatusCode == http.StatusNotModified {
  3209. if res.Body != nil {
  3210. res.Body.Close()
  3211. }
  3212. return nil, &googleapi.Error{
  3213. Code: res.StatusCode,
  3214. Header: res.Header,
  3215. }
  3216. }
  3217. if err != nil {
  3218. return nil, err
  3219. }
  3220. defer googleapi.CloseBody(res)
  3221. if err := googleapi.CheckResponse(res); err != nil {
  3222. return nil, err
  3223. }
  3224. ret := &ListJobsResponse{
  3225. ServerResponse: googleapi.ServerResponse{
  3226. Header: res.Header,
  3227. HTTPStatusCode: res.StatusCode,
  3228. },
  3229. }
  3230. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  3231. return nil, err
  3232. }
  3233. return ret, nil
  3234. // {
  3235. // "description": "List the jobs of a project",
  3236. // "httpMethod": "GET",
  3237. // "id": "dataflow.projects.jobs.list",
  3238. // "parameterOrder": [
  3239. // "projectId"
  3240. // ],
  3241. // "parameters": {
  3242. // "pageSize": {
  3243. // "description": "If there are many jobs, limit response to at most this many. The actual number of jobs returned will be the lesser of max_responses and an unspecified server-defined limit.",
  3244. // "format": "int32",
  3245. // "location": "query",
  3246. // "type": "integer"
  3247. // },
  3248. // "pageToken": {
  3249. // "description": "Set this to the 'next_page_token' field of a previous response to request additional results in a long list.",
  3250. // "location": "query",
  3251. // "type": "string"
  3252. // },
  3253. // "projectId": {
  3254. // "description": "The project which owns the jobs.",
  3255. // "location": "path",
  3256. // "required": true,
  3257. // "type": "string"
  3258. // },
  3259. // "view": {
  3260. // "description": "Level of information requested in response. Default is SUMMARY.",
  3261. // "enum": [
  3262. // "JOB_VIEW_UNKNOWN",
  3263. // "JOB_VIEW_SUMMARY",
  3264. // "JOB_VIEW_ALL"
  3265. // ],
  3266. // "location": "query",
  3267. // "type": "string"
  3268. // }
  3269. // },
  3270. // "path": "v1b3/projects/{projectId}/jobs",
  3271. // "response": {
  3272. // "$ref": "ListJobsResponse"
  3273. // },
  3274. // "scopes": [
  3275. // "https://www.googleapis.com/auth/cloud-platform",
  3276. // "https://www.googleapis.com/auth/userinfo.email"
  3277. // ]
  3278. // }
  3279. }
  3280. // Pages invokes f for each page of results.
  3281. // A non-nil error returned from f will halt the iteration.
  3282. // The provided context supersedes any context provided to the Context method.
  3283. func (c *ProjectsJobsListCall) Pages(ctx context.Context, f func(*ListJobsResponse) error) error {
  3284. c.ctx_ = ctx
  3285. defer c.PageToken(c.urlParams_.Get("pageToken")) // reset paging to original point
  3286. for {
  3287. x, err := c.Do()
  3288. if err != nil {
  3289. return err
  3290. }
  3291. if err := f(x); err != nil {
  3292. return err
  3293. }
  3294. if x.NextPageToken == "" {
  3295. return nil
  3296. }
  3297. c.PageToken(x.NextPageToken)
  3298. }
  3299. }
  3300. // method id "dataflow.projects.jobs.update":
  3301. type ProjectsJobsUpdateCall struct {
  3302. s *Service
  3303. projectId string
  3304. jobId string
  3305. job *Job
  3306. urlParams_ gensupport.URLParams
  3307. ctx_ context.Context
  3308. }
  3309. // Update: Updates the state of an existing dataflow job.
  3310. func (r *ProjectsJobsService) Update(projectId string, jobId string, job *Job) *ProjectsJobsUpdateCall {
  3311. c := &ProjectsJobsUpdateCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  3312. c.projectId = projectId
  3313. c.jobId = jobId
  3314. c.job = job
  3315. return c
  3316. }
  3317. // Fields allows partial responses to be retrieved. See
  3318. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  3319. // for more information.
  3320. func (c *ProjectsJobsUpdateCall) Fields(s ...googleapi.Field) *ProjectsJobsUpdateCall {
  3321. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  3322. return c
  3323. }
  3324. // Context sets the context to be used in this call's Do method. Any
  3325. // pending HTTP request will be aborted if the provided context is
  3326. // canceled.
  3327. func (c *ProjectsJobsUpdateCall) Context(ctx context.Context) *ProjectsJobsUpdateCall {
  3328. c.ctx_ = ctx
  3329. return c
  3330. }
  3331. func (c *ProjectsJobsUpdateCall) doRequest(alt string) (*http.Response, error) {
  3332. var body io.Reader = nil
  3333. body, err := googleapi.WithoutDataWrapper.JSONReader(c.job)
  3334. if err != nil {
  3335. return nil, err
  3336. }
  3337. ctype := "application/json"
  3338. c.urlParams_.Set("alt", alt)
  3339. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/jobs/{jobId}")
  3340. urls += "?" + c.urlParams_.Encode()
  3341. req, _ := http.NewRequest("PUT", urls, body)
  3342. googleapi.Expand(req.URL, map[string]string{
  3343. "projectId": c.projectId,
  3344. "jobId": c.jobId,
  3345. })
  3346. req.Header.Set("Content-Type", ctype)
  3347. req.Header.Set("User-Agent", c.s.userAgent())
  3348. if c.ctx_ != nil {
  3349. return ctxhttp.Do(c.ctx_, c.s.client, req)
  3350. }
  3351. return c.s.client.Do(req)
  3352. }
  3353. // Do executes the "dataflow.projects.jobs.update" call.
  3354. // Exactly one of *Job or error will be non-nil. Any non-2xx status code
  3355. // is an error. Response headers are in either
  3356. // *Job.ServerResponse.Header or (if a response was returned at all) in
  3357. // error.(*googleapi.Error).Header. Use googleapi.IsNotModified to check
  3358. // whether the returned error was because http.StatusNotModified was
  3359. // returned.
  3360. func (c *ProjectsJobsUpdateCall) Do(opts ...googleapi.CallOption) (*Job, error) {
  3361. gensupport.SetOptions(c.urlParams_, opts...)
  3362. res, err := c.doRequest("json")
  3363. if res != nil && res.StatusCode == http.StatusNotModified {
  3364. if res.Body != nil {
  3365. res.Body.Close()
  3366. }
  3367. return nil, &googleapi.Error{
  3368. Code: res.StatusCode,
  3369. Header: res.Header,
  3370. }
  3371. }
  3372. if err != nil {
  3373. return nil, err
  3374. }
  3375. defer googleapi.CloseBody(res)
  3376. if err := googleapi.CheckResponse(res); err != nil {
  3377. return nil, err
  3378. }
  3379. ret := &Job{
  3380. ServerResponse: googleapi.ServerResponse{
  3381. Header: res.Header,
  3382. HTTPStatusCode: res.StatusCode,
  3383. },
  3384. }
  3385. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  3386. return nil, err
  3387. }
  3388. return ret, nil
  3389. // {
  3390. // "description": "Updates the state of an existing dataflow job.",
  3391. // "httpMethod": "PUT",
  3392. // "id": "dataflow.projects.jobs.update",
  3393. // "parameterOrder": [
  3394. // "projectId",
  3395. // "jobId"
  3396. // ],
  3397. // "parameters": {
  3398. // "jobId": {
  3399. // "description": "Identifies a single job.",
  3400. // "location": "path",
  3401. // "required": true,
  3402. // "type": "string"
  3403. // },
  3404. // "projectId": {
  3405. // "description": "The project which owns the job.",
  3406. // "location": "path",
  3407. // "required": true,
  3408. // "type": "string"
  3409. // }
  3410. // },
  3411. // "path": "v1b3/projects/{projectId}/jobs/{jobId}",
  3412. // "request": {
  3413. // "$ref": "Job"
  3414. // },
  3415. // "response": {
  3416. // "$ref": "Job"
  3417. // },
  3418. // "scopes": [
  3419. // "https://www.googleapis.com/auth/cloud-platform",
  3420. // "https://www.googleapis.com/auth/userinfo.email"
  3421. // ]
  3422. // }
  3423. }
  3424. // method id "dataflow.projects.jobs.messages.list":
  3425. type ProjectsJobsMessagesListCall struct {
  3426. s *Service
  3427. projectId string
  3428. jobId string
  3429. urlParams_ gensupport.URLParams
  3430. ifNoneMatch_ string
  3431. ctx_ context.Context
  3432. }
  3433. // List: Request the job status.
  3434. func (r *ProjectsJobsMessagesService) List(projectId string, jobId string) *ProjectsJobsMessagesListCall {
  3435. c := &ProjectsJobsMessagesListCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  3436. c.projectId = projectId
  3437. c.jobId = jobId
  3438. return c
  3439. }
  3440. // EndTime sets the optional parameter "endTime": Return only messages
  3441. // with timestamps < end_time. The default is now (i.e. return up to the
  3442. // latest messages available).
  3443. func (c *ProjectsJobsMessagesListCall) EndTime(endTime string) *ProjectsJobsMessagesListCall {
  3444. c.urlParams_.Set("endTime", endTime)
  3445. return c
  3446. }
  3447. // MinimumImportance sets the optional parameter "minimumImportance":
  3448. // Filter to only get messages with importance >= level
  3449. //
  3450. // Possible values:
  3451. // "JOB_MESSAGE_IMPORTANCE_UNKNOWN"
  3452. // "JOB_MESSAGE_DEBUG"
  3453. // "JOB_MESSAGE_DETAILED"
  3454. // "JOB_MESSAGE_BASIC"
  3455. // "JOB_MESSAGE_WARNING"
  3456. // "JOB_MESSAGE_ERROR"
  3457. func (c *ProjectsJobsMessagesListCall) MinimumImportance(minimumImportance string) *ProjectsJobsMessagesListCall {
  3458. c.urlParams_.Set("minimumImportance", minimumImportance)
  3459. return c
  3460. }
  3461. // PageSize sets the optional parameter "pageSize": If specified,
  3462. // determines the maximum number of messages to return. If unspecified,
  3463. // the service may choose an appropriate default, or may return an
  3464. // arbitrarily large number of results.
  3465. func (c *ProjectsJobsMessagesListCall) PageSize(pageSize int64) *ProjectsJobsMessagesListCall {
  3466. c.urlParams_.Set("pageSize", fmt.Sprint(pageSize))
  3467. return c
  3468. }
  3469. // PageToken sets the optional parameter "pageToken": If supplied, this
  3470. // should be the value of next_page_token returned by an earlier call.
  3471. // This will cause the next page of results to be returned.
  3472. func (c *ProjectsJobsMessagesListCall) PageToken(pageToken string) *ProjectsJobsMessagesListCall {
  3473. c.urlParams_.Set("pageToken", pageToken)
  3474. return c
  3475. }
  3476. // StartTime sets the optional parameter "startTime": If specified,
  3477. // return only messages with timestamps >= start_time. The default is
  3478. // the job creation time (i.e. beginning of messages).
  3479. func (c *ProjectsJobsMessagesListCall) StartTime(startTime string) *ProjectsJobsMessagesListCall {
  3480. c.urlParams_.Set("startTime", startTime)
  3481. return c
  3482. }
  3483. // Fields allows partial responses to be retrieved. See
  3484. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  3485. // for more information.
  3486. func (c *ProjectsJobsMessagesListCall) Fields(s ...googleapi.Field) *ProjectsJobsMessagesListCall {
  3487. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  3488. return c
  3489. }
  3490. // IfNoneMatch sets the optional parameter which makes the operation
  3491. // fail if the object's ETag matches the given value. This is useful for
  3492. // getting updates only after the object has changed since the last
  3493. // request. Use googleapi.IsNotModified to check whether the response
  3494. // error from Do is the result of In-None-Match.
  3495. func (c *ProjectsJobsMessagesListCall) IfNoneMatch(entityTag string) *ProjectsJobsMessagesListCall {
  3496. c.ifNoneMatch_ = entityTag
  3497. return c
  3498. }
  3499. // Context sets the context to be used in this call's Do method. Any
  3500. // pending HTTP request will be aborted if the provided context is
  3501. // canceled.
  3502. func (c *ProjectsJobsMessagesListCall) Context(ctx context.Context) *ProjectsJobsMessagesListCall {
  3503. c.ctx_ = ctx
  3504. return c
  3505. }
  3506. func (c *ProjectsJobsMessagesListCall) doRequest(alt string) (*http.Response, error) {
  3507. var body io.Reader = nil
  3508. c.urlParams_.Set("alt", alt)
  3509. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/jobs/{jobId}/messages")
  3510. urls += "?" + c.urlParams_.Encode()
  3511. req, _ := http.NewRequest("GET", urls, body)
  3512. googleapi.Expand(req.URL, map[string]string{
  3513. "projectId": c.projectId,
  3514. "jobId": c.jobId,
  3515. })
  3516. req.Header.Set("User-Agent", c.s.userAgent())
  3517. if c.ifNoneMatch_ != "" {
  3518. req.Header.Set("If-None-Match", c.ifNoneMatch_)
  3519. }
  3520. if c.ctx_ != nil {
  3521. return ctxhttp.Do(c.ctx_, c.s.client, req)
  3522. }
  3523. return c.s.client.Do(req)
  3524. }
  3525. // Do executes the "dataflow.projects.jobs.messages.list" call.
  3526. // Exactly one of *ListJobMessagesResponse or error will be non-nil. Any
  3527. // non-2xx status code is an error. Response headers are in either
  3528. // *ListJobMessagesResponse.ServerResponse.Header or (if a response was
  3529. // returned at all) in error.(*googleapi.Error).Header. Use
  3530. // googleapi.IsNotModified to check whether the returned error was
  3531. // because http.StatusNotModified was returned.
  3532. func (c *ProjectsJobsMessagesListCall) Do(opts ...googleapi.CallOption) (*ListJobMessagesResponse, error) {
  3533. gensupport.SetOptions(c.urlParams_, opts...)
  3534. res, err := c.doRequest("json")
  3535. if res != nil && res.StatusCode == http.StatusNotModified {
  3536. if res.Body != nil {
  3537. res.Body.Close()
  3538. }
  3539. return nil, &googleapi.Error{
  3540. Code: res.StatusCode,
  3541. Header: res.Header,
  3542. }
  3543. }
  3544. if err != nil {
  3545. return nil, err
  3546. }
  3547. defer googleapi.CloseBody(res)
  3548. if err := googleapi.CheckResponse(res); err != nil {
  3549. return nil, err
  3550. }
  3551. ret := &ListJobMessagesResponse{
  3552. ServerResponse: googleapi.ServerResponse{
  3553. Header: res.Header,
  3554. HTTPStatusCode: res.StatusCode,
  3555. },
  3556. }
  3557. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  3558. return nil, err
  3559. }
  3560. return ret, nil
  3561. // {
  3562. // "description": "Request the job status.",
  3563. // "httpMethod": "GET",
  3564. // "id": "dataflow.projects.jobs.messages.list",
  3565. // "parameterOrder": [
  3566. // "projectId",
  3567. // "jobId"
  3568. // ],
  3569. // "parameters": {
  3570. // "endTime": {
  3571. // "description": "Return only messages with timestamps \u003c end_time. The default is now (i.e. return up to the latest messages available).",
  3572. // "location": "query",
  3573. // "type": "string"
  3574. // },
  3575. // "jobId": {
  3576. // "description": "The job to get messages about.",
  3577. // "location": "path",
  3578. // "required": true,
  3579. // "type": "string"
  3580. // },
  3581. // "minimumImportance": {
  3582. // "description": "Filter to only get messages with importance \u003e= level",
  3583. // "enum": [
  3584. // "JOB_MESSAGE_IMPORTANCE_UNKNOWN",
  3585. // "JOB_MESSAGE_DEBUG",
  3586. // "JOB_MESSAGE_DETAILED",
  3587. // "JOB_MESSAGE_BASIC",
  3588. // "JOB_MESSAGE_WARNING",
  3589. // "JOB_MESSAGE_ERROR"
  3590. // ],
  3591. // "location": "query",
  3592. // "type": "string"
  3593. // },
  3594. // "pageSize": {
  3595. // "description": "If specified, determines the maximum number of messages to return. If unspecified, the service may choose an appropriate default, or may return an arbitrarily large number of results.",
  3596. // "format": "int32",
  3597. // "location": "query",
  3598. // "type": "integer"
  3599. // },
  3600. // "pageToken": {
  3601. // "description": "If supplied, this should be the value of next_page_token returned by an earlier call. This will cause the next page of results to be returned.",
  3602. // "location": "query",
  3603. // "type": "string"
  3604. // },
  3605. // "projectId": {
  3606. // "description": "A project id.",
  3607. // "location": "path",
  3608. // "required": true,
  3609. // "type": "string"
  3610. // },
  3611. // "startTime": {
  3612. // "description": "If specified, return only messages with timestamps \u003e= start_time. The default is the job creation time (i.e. beginning of messages).",
  3613. // "location": "query",
  3614. // "type": "string"
  3615. // }
  3616. // },
  3617. // "path": "v1b3/projects/{projectId}/jobs/{jobId}/messages",
  3618. // "response": {
  3619. // "$ref": "ListJobMessagesResponse"
  3620. // },
  3621. // "scopes": [
  3622. // "https://www.googleapis.com/auth/cloud-platform",
  3623. // "https://www.googleapis.com/auth/userinfo.email"
  3624. // ]
  3625. // }
  3626. }
  3627. // Pages invokes f for each page of results.
  3628. // A non-nil error returned from f will halt the iteration.
  3629. // The provided context supersedes any context provided to the Context method.
  3630. func (c *ProjectsJobsMessagesListCall) Pages(ctx context.Context, f func(*ListJobMessagesResponse) error) error {
  3631. c.ctx_ = ctx
  3632. defer c.PageToken(c.urlParams_.Get("pageToken")) // reset paging to original point
  3633. for {
  3634. x, err := c.Do()
  3635. if err != nil {
  3636. return err
  3637. }
  3638. if err := f(x); err != nil {
  3639. return err
  3640. }
  3641. if x.NextPageToken == "" {
  3642. return nil
  3643. }
  3644. c.PageToken(x.NextPageToken)
  3645. }
  3646. }
  3647. // method id "dataflow.projects.jobs.workItems.lease":
  3648. type ProjectsJobsWorkItemsLeaseCall struct {
  3649. s *Service
  3650. projectId string
  3651. jobId string
  3652. leaseworkitemrequest *LeaseWorkItemRequest
  3653. urlParams_ gensupport.URLParams
  3654. ctx_ context.Context
  3655. }
  3656. // Lease: Leases a dataflow WorkItem to run.
  3657. func (r *ProjectsJobsWorkItemsService) Lease(projectId string, jobId string, leaseworkitemrequest *LeaseWorkItemRequest) *ProjectsJobsWorkItemsLeaseCall {
  3658. c := &ProjectsJobsWorkItemsLeaseCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  3659. c.projectId = projectId
  3660. c.jobId = jobId
  3661. c.leaseworkitemrequest = leaseworkitemrequest
  3662. return c
  3663. }
  3664. // Fields allows partial responses to be retrieved. See
  3665. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  3666. // for more information.
  3667. func (c *ProjectsJobsWorkItemsLeaseCall) Fields(s ...googleapi.Field) *ProjectsJobsWorkItemsLeaseCall {
  3668. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  3669. return c
  3670. }
  3671. // Context sets the context to be used in this call's Do method. Any
  3672. // pending HTTP request will be aborted if the provided context is
  3673. // canceled.
  3674. func (c *ProjectsJobsWorkItemsLeaseCall) Context(ctx context.Context) *ProjectsJobsWorkItemsLeaseCall {
  3675. c.ctx_ = ctx
  3676. return c
  3677. }
  3678. func (c *ProjectsJobsWorkItemsLeaseCall) doRequest(alt string) (*http.Response, error) {
  3679. var body io.Reader = nil
  3680. body, err := googleapi.WithoutDataWrapper.JSONReader(c.leaseworkitemrequest)
  3681. if err != nil {
  3682. return nil, err
  3683. }
  3684. ctype := "application/json"
  3685. c.urlParams_.Set("alt", alt)
  3686. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/jobs/{jobId}/workItems:lease")
  3687. urls += "?" + c.urlParams_.Encode()
  3688. req, _ := http.NewRequest("POST", urls, body)
  3689. googleapi.Expand(req.URL, map[string]string{
  3690. "projectId": c.projectId,
  3691. "jobId": c.jobId,
  3692. })
  3693. req.Header.Set("Content-Type", ctype)
  3694. req.Header.Set("User-Agent", c.s.userAgent())
  3695. if c.ctx_ != nil {
  3696. return ctxhttp.Do(c.ctx_, c.s.client, req)
  3697. }
  3698. return c.s.client.Do(req)
  3699. }
  3700. // Do executes the "dataflow.projects.jobs.workItems.lease" call.
  3701. // Exactly one of *LeaseWorkItemResponse or error will be non-nil. Any
  3702. // non-2xx status code is an error. Response headers are in either
  3703. // *LeaseWorkItemResponse.ServerResponse.Header or (if a response was
  3704. // returned at all) in error.(*googleapi.Error).Header. Use
  3705. // googleapi.IsNotModified to check whether the returned error was
  3706. // because http.StatusNotModified was returned.
  3707. func (c *ProjectsJobsWorkItemsLeaseCall) Do(opts ...googleapi.CallOption) (*LeaseWorkItemResponse, error) {
  3708. gensupport.SetOptions(c.urlParams_, opts...)
  3709. res, err := c.doRequest("json")
  3710. if res != nil && res.StatusCode == http.StatusNotModified {
  3711. if res.Body != nil {
  3712. res.Body.Close()
  3713. }
  3714. return nil, &googleapi.Error{
  3715. Code: res.StatusCode,
  3716. Header: res.Header,
  3717. }
  3718. }
  3719. if err != nil {
  3720. return nil, err
  3721. }
  3722. defer googleapi.CloseBody(res)
  3723. if err := googleapi.CheckResponse(res); err != nil {
  3724. return nil, err
  3725. }
  3726. ret := &LeaseWorkItemResponse{
  3727. ServerResponse: googleapi.ServerResponse{
  3728. Header: res.Header,
  3729. HTTPStatusCode: res.StatusCode,
  3730. },
  3731. }
  3732. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  3733. return nil, err
  3734. }
  3735. return ret, nil
  3736. // {
  3737. // "description": "Leases a dataflow WorkItem to run.",
  3738. // "httpMethod": "POST",
  3739. // "id": "dataflow.projects.jobs.workItems.lease",
  3740. // "parameterOrder": [
  3741. // "projectId",
  3742. // "jobId"
  3743. // ],
  3744. // "parameters": {
  3745. // "jobId": {
  3746. // "description": "Identifies the workflow job this worker belongs to.",
  3747. // "location": "path",
  3748. // "required": true,
  3749. // "type": "string"
  3750. // },
  3751. // "projectId": {
  3752. // "description": "Identifies the project this worker belongs to.",
  3753. // "location": "path",
  3754. // "required": true,
  3755. // "type": "string"
  3756. // }
  3757. // },
  3758. // "path": "v1b3/projects/{projectId}/jobs/{jobId}/workItems:lease",
  3759. // "request": {
  3760. // "$ref": "LeaseWorkItemRequest"
  3761. // },
  3762. // "response": {
  3763. // "$ref": "LeaseWorkItemResponse"
  3764. // },
  3765. // "scopes": [
  3766. // "https://www.googleapis.com/auth/cloud-platform",
  3767. // "https://www.googleapis.com/auth/userinfo.email"
  3768. // ]
  3769. // }
  3770. }
  3771. // method id "dataflow.projects.jobs.workItems.reportStatus":
  3772. type ProjectsJobsWorkItemsReportStatusCall struct {
  3773. s *Service
  3774. projectId string
  3775. jobId string
  3776. reportworkitemstatusrequest *ReportWorkItemStatusRequest
  3777. urlParams_ gensupport.URLParams
  3778. ctx_ context.Context
  3779. }
  3780. // ReportStatus: Reports the status of dataflow WorkItems leased by a
  3781. // worker.
  3782. func (r *ProjectsJobsWorkItemsService) ReportStatus(projectId string, jobId string, reportworkitemstatusrequest *ReportWorkItemStatusRequest) *ProjectsJobsWorkItemsReportStatusCall {
  3783. c := &ProjectsJobsWorkItemsReportStatusCall{s: r.s, urlParams_: make(gensupport.URLParams)}
  3784. c.projectId = projectId
  3785. c.jobId = jobId
  3786. c.reportworkitemstatusrequest = reportworkitemstatusrequest
  3787. return c
  3788. }
  3789. // Fields allows partial responses to be retrieved. See
  3790. // https://developers.google.com/gdata/docs/2.0/basics#PartialResponse
  3791. // for more information.
  3792. func (c *ProjectsJobsWorkItemsReportStatusCall) Fields(s ...googleapi.Field) *ProjectsJobsWorkItemsReportStatusCall {
  3793. c.urlParams_.Set("fields", googleapi.CombineFields(s))
  3794. return c
  3795. }
  3796. // Context sets the context to be used in this call's Do method. Any
  3797. // pending HTTP request will be aborted if the provided context is
  3798. // canceled.
  3799. func (c *ProjectsJobsWorkItemsReportStatusCall) Context(ctx context.Context) *ProjectsJobsWorkItemsReportStatusCall {
  3800. c.ctx_ = ctx
  3801. return c
  3802. }
  3803. func (c *ProjectsJobsWorkItemsReportStatusCall) doRequest(alt string) (*http.Response, error) {
  3804. var body io.Reader = nil
  3805. body, err := googleapi.WithoutDataWrapper.JSONReader(c.reportworkitemstatusrequest)
  3806. if err != nil {
  3807. return nil, err
  3808. }
  3809. ctype := "application/json"
  3810. c.urlParams_.Set("alt", alt)
  3811. urls := googleapi.ResolveRelative(c.s.BasePath, "v1b3/projects/{projectId}/jobs/{jobId}/workItems:reportStatus")
  3812. urls += "?" + c.urlParams_.Encode()
  3813. req, _ := http.NewRequest("POST", urls, body)
  3814. googleapi.Expand(req.URL, map[string]string{
  3815. "projectId": c.projectId,
  3816. "jobId": c.jobId,
  3817. })
  3818. req.Header.Set("Content-Type", ctype)
  3819. req.Header.Set("User-Agent", c.s.userAgent())
  3820. if c.ctx_ != nil {
  3821. return ctxhttp.Do(c.ctx_, c.s.client, req)
  3822. }
  3823. return c.s.client.Do(req)
  3824. }
  3825. // Do executes the "dataflow.projects.jobs.workItems.reportStatus" call.
  3826. // Exactly one of *ReportWorkItemStatusResponse or error will be
  3827. // non-nil. Any non-2xx status code is an error. Response headers are in
  3828. // either *ReportWorkItemStatusResponse.ServerResponse.Header or (if a
  3829. // response was returned at all) in error.(*googleapi.Error).Header. Use
  3830. // googleapi.IsNotModified to check whether the returned error was
  3831. // because http.StatusNotModified was returned.
  3832. func (c *ProjectsJobsWorkItemsReportStatusCall) Do(opts ...googleapi.CallOption) (*ReportWorkItemStatusResponse, error) {
  3833. gensupport.SetOptions(c.urlParams_, opts...)
  3834. res, err := c.doRequest("json")
  3835. if res != nil && res.StatusCode == http.StatusNotModified {
  3836. if res.Body != nil {
  3837. res.Body.Close()
  3838. }
  3839. return nil, &googleapi.Error{
  3840. Code: res.StatusCode,
  3841. Header: res.Header,
  3842. }
  3843. }
  3844. if err != nil {
  3845. return nil, err
  3846. }
  3847. defer googleapi.CloseBody(res)
  3848. if err := googleapi.CheckResponse(res); err != nil {
  3849. return nil, err
  3850. }
  3851. ret := &ReportWorkItemStatusResponse{
  3852. ServerResponse: googleapi.ServerResponse{
  3853. Header: res.Header,
  3854. HTTPStatusCode: res.StatusCode,
  3855. },
  3856. }
  3857. if err := json.NewDecoder(res.Body).Decode(&ret); err != nil {
  3858. return nil, err
  3859. }
  3860. return ret, nil
  3861. // {
  3862. // "description": "Reports the status of dataflow WorkItems leased by a worker.",
  3863. // "httpMethod": "POST",
  3864. // "id": "dataflow.projects.jobs.workItems.reportStatus",
  3865. // "parameterOrder": [
  3866. // "projectId",
  3867. // "jobId"
  3868. // ],
  3869. // "parameters": {
  3870. // "jobId": {
  3871. // "description": "The job which the WorkItem is part of.",
  3872. // "location": "path",
  3873. // "required": true,
  3874. // "type": "string"
  3875. // },
  3876. // "projectId": {
  3877. // "description": "The project which owns the WorkItem's job.",
  3878. // "location": "path",
  3879. // "required": true,
  3880. // "type": "string"
  3881. // }
  3882. // },
  3883. // "path": "v1b3/projects/{projectId}/jobs/{jobId}/workItems:reportStatus",
  3884. // "request": {
  3885. // "$ref": "ReportWorkItemStatusRequest"
  3886. // },
  3887. // "response": {
  3888. // "$ref": "ReportWorkItemStatusResponse"
  3889. // },
  3890. // "scopes": [
  3891. // "https://www.googleapis.com/auth/cloud-platform",
  3892. // "https://www.googleapis.com/auth/userinfo.email"
  3893. // ]
  3894. // }
  3895. }