filebeat.inputs:
- type: httpjson
  interval: "1h"
  request.url: https://proofpointisolation.com/api/v2/reporting/useage-data/
  request.method: GET
  request.transforms:
    - set:
        target: url.params.to
        value: '[[formatDate (now)]]'
    - set:
        target: url.params.from
        value: '[[ formatDate (now (parseDuration "-1h")) ]]'
    - set:
        target: url.params.key
        value: 'your_api_key'
    - set:
        target: url.params.pageSize
        value: '100'
  response.split:
    target: body.data
    type: array
    keep_parent: false
  response.pagination:
    - set:
        target: url.value
        value: '[[ .last_response.url.value]]' 
    - set:
        target: url.params.jobId
        value: '[[.last_response.body.jobId]]'
        fail_on_template_error: true
    - set:
        target: url.params.pageToken
        value: '[[.last_response.body.pageToken]]'
        fail_on_template_error: true
  processors:
    - decode_json_fields:
        fields: ["message"]
        target: "json"

output.console:
  pretty: true

if(ctx.field_name != null){
  ArrayList lower_array = new ArrayList();
  for (item in ctx.field_name){
    ArrayList temp = new ArrayList();
    for(map_entry in item.entrySet()){
      temp.add([map_entry.getKey().toLowerCase():map_entry.getValue()])
    }
    lower_array.add(temp);
  }
  ctx.field_name = lower_array;
}

function Get-ListItems{
  param(
      [srtring]$sharepointSite,
      [srtring]$listTitle,
      [srtring]$selectedFields
    )
  $itemArry =@()
  $pageSize = 5000
  $pageId = 0
  $header = @{'Accept' = 'application/json; odata=verbose'}

  if ($selectedFields.Length -gt 0){
    $selectStatement = "&`$SELECT=$($selectedFields)"
  }

  while ($pageId){
    $uri = "$($sharepointSite)/_api/Web/Lists/GetByTitle('$($listTitle)')/Items?`$skiptoken=Paged=True%26p_ID=$($pageId)&`$top=$($pageSize)$($selectStatement)"

    #Get list Get-ListItems
    $response = Invoke-RestMethod -Method Get -Uri $uri -Headers $header -UseDefaultCredentials:$true
    $data = $response.ToString().Replace("""ID""","_ID") | ConvertFrom-Json

    if($data.d.results.Count -gt 0){
      #Add to the Array
      $itemsArray += $data.d.results
      #Get last id in the results set for next starting sharepointSite
      $pageId = $data.d.results[-1].id
    }
    else{
      $pageId =-1
    }
  }
  return $itemArry

}

powershell.exe -nop -exec bypass