OpenAI #FtJob: イベント一覧

// Script Example of Business Process Automation
// for 'engine type: 3' ("GraalJS standard mode")
// cf. 'engine type: 2' ("GraalJS Nashorn compatible mode") (renamed from "GraalJS" at 20230526)

//////// START "main()" /////////////////////////////////////////////////////////////////

main();
function main(){ 

////// == Config Retrieving / 工程コンフィグの参照 ==
const strAuthzSetting     = configs.get( "AuthzConfU1" );             /// REQUIRED
  engine.log( " AutomatedTask Config: Authz Setting: " + strAuthzSetting );
const strOrgId            = configs.get( "StrConfU2" );               // NotRequired
  engine.log( " AutomatedTask Config: OpenAI-Organization: " + strOrgId );
const strJobId            = configs.get( "StrConfA1" );               /// REQUIRED (updated 20230822)
  if( strJobId          === "" ){
    throw new Error( "\n AutomatedTask ConfigError:" +
                     " Config {A1: Job Id} must be non-empty string \n" );
  }
const numPocketEvents     = configs.getObject( "SelectConfB1" );      // NotRequired
const strPocketEventsTsv  = configs.getObject( "SelectConfB3" );      // NotRequired



////// == Data Retrieving / ワークフローデータの参照 ==
// (Nothing. Retrieved via Expression Language in Config Retrieving)



////// == Calculating / 演算 ==

//// OpenAI API > Documentation > API REFERENCE > Fine-tuning > List fine-tuning events
//// https://platform.openai.com/docs/api-reference/fine-tuning/list-events

/// prepare request1
let request1Uri = "https://api.openai.com/v1/fine_tuning/jobs/" + strJobId + "/events";
let request1 = httpClient.begin(); // HttpRequestWrapper
    request1 = request1.authSetting( strAuthzSetting ); // with "Authorization: Bearer XX"
    if ( strOrgId !== "" ){
      request1 = request1.header( "OpenAI-Organization", strOrgId );
    }

/// try request1
const response1     = request1.get( request1Uri ); // HttpResponseWrapper
engine.log( " AutomatedTask ApiRequest1 Start: " + request1Uri );
const response1Code = response1.getStatusCode() + ""; // JavaNum to string
const response1Body = response1.getResponseAsString();
engine.log( " AutomatedTask ApiResponse1 Status: " + response1Code );
if( response1Code !== "200"){
  throw new Error( "\n AutomatedTask UnexpectedResponseError: " +
                    response1Code + "\n" + response1Body + "\n" );
}

/* engine.log( response1Body ); // debug
{
  "object":"list",
  "data":[
    {
      "object":"fine_tuning.job.event",
      "id":"ftevent-cIKDbw3CRLr7IwYsTBEKS2qn",
      "created_at":1692846984,
      "level":"warn",
      "message":"Fine tuning process stopping due to job cancellation",
      "data":null,
      "type":"message"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-yqPcSyX5qXQ3ouMqWb4XraHR",
      "created_at":1692846231,
      "level":"info",
      "message":"Fine tuning job started",
      "data":null,
      "type":"message"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-piJrXGJFrAcvRpjgk7f1ozb8",
      "created_at":1692846230,
      "level":"info",
      "message":"Created fine-tune: ftjob-8X0qeoZozDCd9fmRcXLLrk30",
      "data":null,
      "type":"message"
    }
  ],
  "has_more":false
}

 v( sanity check case - "train loss" should decrease, "token accuracy" should increase )v

{
  "object":"list",
  "data":[
    {
      "object":"fine_tuning.job.event",
      "id":"ftevent-6JRdoZOAEP42fRMT4fAat16z",
      "created_at":1692775663,
      "level":"info",
      "message":"Fine-tuning job successfully completed",
      "data":null,
      "type":"message"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-9o8pLa7zmnzMrpxzYt6D08Ga",
      "created_at":1692775661,
      "level":"info",
      "message":"New fine-tuned model created: ft:gpt-3.5-turbo-0613:questetra::7qcZBkZT",
      "data":null,
      "type":"message"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-A1JYs6eWQMzB6OYewerUwyqh",
      "created_at":1692775657,
      "level":"info",
      "message":"Step 100: training loss=0.46",
      "data":{
        "step":100,
        "train_loss":0.4633517563343048,
        "train_mean_token_accuracy":0.899328887462616
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-nScwPTux35QHd9xL1oo0bAk0",
      "created_at":1692775647,
      "level":"info",
      "message":"Step 90: training loss=0.92",
      "data":{
        "step":90,
        "train_loss":0.9216997623443604,
        "train_mean_token_accuracy":0.7945205569267273
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-FL47WDAJ0qJYgaZDdzRsJRJi",
      "created_at":1692775635,
      "level":"info",
      "message":"Step 80: training loss=0.87",
      "data":{
        "step":80,
        "train_loss":0.8748041987419128,
        "train_mean_token_accuracy":0.7610062956809998
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-j4j9nB5sJwysnhs10pDiDEJK",
      "created_at":1692775623,
      "level":"info",
      "message":"Step 70: training loss=0.37",
      "data":{
        "step":70,
        "train_loss":0.37020447850227356,
        "train_mean_token_accuracy":0.8873239159584045
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-sDGXzDpDsNeLSNSGqLCnCBLH",
      "created_at":1692775613,
      "level":"info",
      "message":"Step 60: training loss=0.72",
      "data":{
        "step":60,
        "train_loss":0.7190837860107422,
        "train_mean_token_accuracy":0.739130437374115
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-Ulle9AK7kjK6Z3BYBLWRQPi9",
      "created_at":1692775601,
      "level":"info",
      "message":"Step 50: training loss=0.84",
      "data":{
        "step":50,
        "train_loss":0.8407266736030579,
        "train_mean_token_accuracy":0.7816091775894165
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-Cq063vJgvuopfWDGQGgRcPJU",
      "created_at":1692775591,
      "level":"info",
      "message":"Step 40: training loss=0.74",
      "data":{
        "step":40,
        "train_loss":0.7438284158706665,
        "train_mean_token_accuracy":0.8324607610702515
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-QngoPDBAGw8qrX3sv0yJ2xZk",
      "created_at":1692775579,
      "level":"info",
      "message":"Step 30: training loss=1.17",
      "data":{
        "step":30,
        "train_loss":1.170945644378662,
        "train_mean_token_accuracy":0.7362204790115356
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-73ZWTwvIFAG7Wl5TFjaPxrDp",
      "created_at":1692775567,
      "level":"info",
      "message":"Step 20: training loss=1.53",
      "data":{
        "step":20,
        "train_loss":1.5336471796035767,
        "train_mean_token_accuracy":0.6594203114509583
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-adP31yqzjtX08xrjlXLFrBvj",
      "created_at":1692775557,
      "level":"info",
      "message":"Step 10: training loss=1.16",
      "data":{
        "step":10,
        "train_loss":1.155320167541504,
        "train_mean_token_accuracy":0.801047146320343
      },
      "type":"metrics"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-EA8B1acesYMoe1Xyn8NBiPZK",
      "created_at":1692775088,
      "level":"info",
      "message":"Fine tuning job started",
      "data":null,
      "type":"message"
    },{
      "object":"fine_tuning.job.event",
      "id":"ftevent-NUJYdDjGv0A6wRALH1dYqq3Z",
      "created_at":1692775087,
      "level":"info",
      "message":"Created fine-tune: ftjob-XWe29EU2BqUQSEakRDYUqGVX",
      "data":null,
      "type":"message"
    }
  ],
  "has_more":false
}
*/

/// parse response1
const response1Obj = JSON.parse( response1Body );
let numEvents = response1Obj?.data?.length ?? 0;
let arrEventsData = [];
for ( let i = 0; i < numEvents; i++ ) { // created time, event level, event message. (upto 20)
  const dateTmpCreate = new Date ( response1Obj.data[i].created_at * 1000 );
  arrEventsData.push ( 
    dateTmpCreate.toISOString() + '\t' +
    response1Obj.data[i].level + '\t' +
    response1Obj.data[i].message
  );
}



////// == Data Updating / ワークフローデータへの代入 ==
if( numPocketEvents !== null ){
  engine.setData( numPocketEvents, new java.math.BigDecimal( numEvents ) );
}
if( strPocketEventsTsv !== null ){
  engine.setData( strPocketEventsTsv, arrEventsData.join('\n') );
}


} //////// END "main()" /////////////////////////////////////////////////////////////////



/*
Notes:
- This "Automated Step" will list the fine-tuning job events by FtJob ID.
    - Example of FtJob ID: "ft-AF1WoRqd3aJAHsqc9NY7iL8F"
- If you place this "Automated Step" in the Workflow diagram, the request will be automatically sent every time the process token arrives.
    - A request is automatically sent to the OpenAI API server. (REST API)
    - The response from the OpenAI API server is automatically parsed.
APPENDIX
- To activate a Workflow App that includes this Automated Step, "HTTP Authz Setting" is required
    - Obtain a "Secret API Key" in advance.
    - Set the key as the communication token in "Token Fixed Value"


Notes-ja:
- この［自動工程］は、FtJob ID でファインチューンJob（微調整ジョブ）の詳細イベントをリストします。
    - Example of FtJob ID: "ft-AF1WoRqd3aJAHsqc9NY7iL8F"
- この［自動工程］をワークフロー図に配置すれば、案件が到達する度にリクエストが自動送信されます。
    - OpenAI API サーバに対してリクエストが自動送出されます。（REST API通信）
    - OpenAI API サーバからのレスポンスが自動保存解析されます。
APPENDIX-ja
- この［アドオン自動工程］を含むワークフローアプリを運用するには［HTTP 認証設定］が必要です。
    - あらかじめ "Secret API Key" を取得しておいてください。
    - "Secret API Key" を通信トークンとしてセットします。［トークン直接指定］
*/