Viewing file:  Aggregator.php (9.14 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
<?php
/*
* @author Andreas Åkre Solberg <andreas.solberg@uninett.no>
* @package simpleSAMLphp
* @version $Id$
*/
class sspmod_statistics_Aggregator {
private $statconfig;
private $statdir;
private $inputfile;
private $statrules;
private $offset;
private $metadata;
private $fromcmdline;
private $starttime;
/**
* Constructor
*/
public function __construct($fromcmdline = FALSE) {
$this->fromcmdline = $fromcmdline;
$this->statconfig = SimpleSAML_Configuration::getConfig('module_statistics.php');
$this->statdir = $this->statconfig->getValue('statdir');
$this->inputfile = $this->statconfig->getValue('inputfile');
$this->statrules = $this->statconfig->getValue('statrules');
$this->timeres = $this->statconfig->getValue('timeres');
$this->offset = $this->statconfig->getValue('offset', 0);
$this->metadata = NULL;
$this->starttime = time();
}
public function dumpConfig() {
echo 'Statistics directory : ' . $this->statdir . "\n";
echo 'Input file : ' . $this->inputfile . "\n";
echo 'Offset : ' . $this->offset . "\n";
}
public function debugInfo() {
echo 'Memory usage : ' . number_format(memory_get_usage() / (1024*1024), 2) . " MB\n";
}
public function loadMetadata() {
$filename = $this->statdir . '/.stat.metadata';
$metadata = NULL;
if (file_exists($filename)) {
$metadata = unserialize(file_get_contents($filename));
}
$this->metadata = $metadata;
}
public function getMetadata() {
return $this->metadata;
}
public function saveMetadata() {
$this->metadata['time'] = time() - $this->starttime;
$this->metadata['memory'] = memory_get_usage();
$this->metadata['lastrun'] = time();
$filename = $this->statdir . '/.stat.metadata';
file_put_contents($filename, serialize($this->metadata), LOCK_EX);
}
public function aggregate($debug = FALSE) {
$this->loadMetadata();
if (!is_dir($this->statdir))
throw new Exception('Statistics module: output dir do not exists [' . $this->statdir . ']');
if (!file_exists($this->inputfile))
throw new Exception('Statistics module: input file do not exists [' . $this->inputfile . ']');
$file = fopen($this->inputfile, 'r');
#$logfile = file($this->inputfile, FILE_IGNORE_NEW_LINES );
$logparser = new sspmod_statistics_LogParser(
$this->statconfig->getValue('datestart', 0), $this->statconfig->getValue('datelength', 15), $this->statconfig->getValue('offsetspan', 44)
);
$datehandler = array(
'default' => new sspmod_statistics_DateHandler($this->offset),
'month' => new sspmod_statistics_DateHandlerMonth($this->offset),
);
$notBefore = 0; $lastRead = 0; $lastlinehash = '-';
if (isset($this->metadata)) {
$notBefore = $this->metadata['notBefore'];
$lastlinehash = $this->metadata['lastlinehash'];
}
$lastlogline = 'sdfsdf';
$lastlineflip = FALSE;
$results = array();
$i = 0;
// Parse through log file, line by line
while (!feof($file)) {
$logline = fgets($file, 4096);
// Continue if STAT is not found on line.
if (!preg_match('/STAT/', $logline)) continue;
$i++; $lastlogline = $logline;
// Parse log, and extract epoch time and rest of content.
$epoch = $logparser->parseEpoch($logline);
$content = $logparser->parseContent($logline);
$action = trim($content[5]);
if ($this->fromcmdline && ($i % 10000) == 0) {
echo("Read line " . $i . "\n");
}
if ($debug) {
echo("----------------------------------------\n");
echo('Log line: ' . $logline . "\n");
echo('Date parse [' . substr($logline, 0, $this->statconfig->getValue('datelength', 15)) . '] to [' . date(DATE_RFC822, $epoch) . ']' . "\n");
print_r($content);
if ($i >= 13) exit;
}
if ($epoch > $lastRead) $lastRead = $epoch;
if ($epoch === $notBefore) {
if(!$lastlineflip) {
if (sha1($logline) === $lastlinehash) {
$lastlineflip = TRUE;
}
continue;
}
}
if ($epoch < $notBefore) continue;
// Iterate all the statrules from config.
foreach ($this->statrules AS $rulename => $rule) {
$type = 'aggregate';
if (array_key_exists('type', $rule)) $type = $rule['type'];
if ($type !== 'aggregate') continue;
foreach($this->timeres AS $tres => $tresconfig ) {
// echo 'Comparing action: [' . $rule['action'] . '] with [' . $action . ']' . "\n";
$dh = 'default';
if (isset($tresconfig['customDateHandler'])) $dh = $tresconfig['customDateHandler'];
$timeslot = $datehandler['default']->toSlot($epoch, $tresconfig['slot']);
$fileslot = $datehandler[$dh]->toSlot($epoch, $tresconfig['fileslot']); //print_r($content);
if (isset($rule['action']) && ($action !== $rule['action'])) continue;
#$difcol = trim($content[$rule['col']]); // echo '[...' . $difcol . '...]';
$difcol = self::getDifCol($content, $rule['col']);
if (!isset($results[$rulename][$tres][$fileslot][$timeslot]['_'])) $results[$rulename][$tres][$fileslot][$timeslot]['_'] = 0;
if (!isset($results[$rulename][$tres][$fileslot][$timeslot][$difcol])) $results[$rulename][$tres][$fileslot][$timeslot][$difcol] = 0;
$results[$rulename][$tres][$fileslot][$timeslot]['_']++;
$results[$rulename][$tres][$fileslot][$timeslot][$difcol]++;
}
}
}
$this->metadata['notBefore'] = $lastRead;
$this->metadata['lastline'] = $lastlogline;
$this->metadata['lastlinehash'] = sha1($lastlogline);
return $results;
}
private static function getDifCol($content, $colrule) {
if (is_int($colrule)) {
return trim($content[$colrule]);
} elseif(is_array($colrule)) {
$difcols = array();
foreach($colrule AS $cr) {
$difcols[] = trim($content[$cr]);
}
return join('|', $difcols);
} else {
return 'NA';
}
}
private function cummulateData($previous, $newdata) {
$dataset = array();
foreach($previous AS $slot => $dataarray) {
if (!array_key_exists($slot, $dataset)) $dataset[$slot] = array();
foreach($dataarray AS $key => $data) {
if (!array_key_exists($key, $dataset[$slot])) $dataset[$slot][$key] = 0;
$dataset[$slot][$key] += $data;
}
}
foreach($newdata AS $slot => $dataarray) {
if (!array_key_exists($slot, $dataset)) $dataset[$slot] = array();
foreach($dataarray AS $key => $data) {
if (!array_key_exists($key, $dataset[$slot])) $dataset[$slot][$key] = 0;
$dataset[$slot][$key] += $data;
}
}
return $dataset;
}
public function store($results) {
// print_r($results); // exit;
$datehandler = array(
'default' => new sspmod_statistics_DateHandler($this->offset),
'month' => new sspmod_statistics_DateHandlerMonth($this->offset),
);
// Iterate the first level of results, which is per rule, as defined in the config.
foreach ($results AS $rulename => $timeresdata) {
// $timeresl = array_keys($timeresdata);
//
// print_r($timeresl); exit;
// Iterate over time resolutions
foreach($timeresdata AS $tres => $resres) {
$dh = 'default';
if (isset($this->timeres[$tres]['customDateHandler'])) $dh = $this->timeres[$tres]['customDateHandler'];
$filenos = array_keys($resres);
$lastfile = $filenos[count($filenos)-1];
// Iterate the second level of results, which is the fileslot.
foreach ($resres AS $fileno => $fileres) {
// Slots that have data.
$slotlist = array_keys($fileres);
// The last slot.
$maxslot = $slotlist[count($slotlist)-1];
#print_r($slotlist);
// Get start and end slot number within the file, based on the fileslot.
$start = (int)$datehandler['default']->toSlot(
$datehandler[$dh]->fromSlot($fileno, $this->timeres[$tres]['fileslot']),
$this->timeres[$tres]['slot']);
$end = (int)$datehandler['default']->toSlot(
$datehandler[$dh]->fromSlot($fileno+1, $this->timeres[$tres]['fileslot']),
$this->timeres[$tres]['slot']);
// echo('from slot ' . $start . ' to slot ' . $end . ' maxslot ' . $maxslot . "\n");
// print_r($slotlist);
// exit;
// Fill in missing entries and sort file results
$filledresult = array();
for ($slot = $start; $slot < $end; $slot++) {
if (array_key_exists($slot, $fileres)) {
$filledresult[$slot] = $fileres[$slot];
} else {
#echo('SLot [' . $slot . '] of [' . $maxslot . ']' . "\n");
if ($lastfile == $fileno && $slot > $maxslot) {
$filledresult[$slot] = array('_' => NULL);
} else {
$filledresult[$slot] = array('_' => 0);
}
}
# print_r($filledresult[$slot]);
# = (isset($fileres[$slot])) ? $fileres[$slot] : array('_' => NULL);
}
// print_r($filledresult); exit;
$filename = $this->statdir . '/' . $rulename . '-' . $tres . '-' . $fileno . '.stat';
if (file_exists($filename)) {
// echo('Reading existing file: ' . $filename . "\n");
$previousData = unserialize(file_get_contents($filename));
$filledresult = $this->cummulateData($previousData, $filledresult);
}
// store file
# echo('Writing to file: ' . $filename . "\n");
file_put_contents($filename, serialize($filledresult), LOCK_EX);
}
}
}
$this->saveMetadata();
}
}
|