Faster User ID reports processing: use ranking query #14551
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,7 +10,8 @@ | |
| use Piwik\Config; | ||
| use Piwik\DataArray; | ||
| use Piwik\DataTable; | ||
| use Piwik\Metrics as PiwikMetrics; | ||
| use Piwik\RankingQuery; | ||
|
|
||
| /** | ||
| * Archiver that aggregates metrics per user ID (user_id field). | ||
|
|
@@ -77,19 +78,32 @@ protected function aggregateUsers() | |
| $userIdFieldName = self::USER_ID_FIELD; | ||
| $visitorIdFieldName = self::VISITOR_ID_FIELD; | ||
|
|
||
| $rankingQueryLimit = $this->getRankingQueryLimit(); | ||
|
|
||
| $rankingQuery = false; | ||
| if ($rankingQueryLimit > 0) { | ||
| $rankingQuery = new RankingQuery($rankingQueryLimit); | ||
| $rankingQuery->setOthersLabel(DataTable::LABEL_SUMMARY_ROW); | ||
| $rankingQuery->addLabelColumn($userIdFieldName); | ||
| $rankingQuery->addLabelColumn($visitorIdFieldName); | ||
| } | ||
|
|
||
| /** @var \Zend_Db_Statement $query */ | ||
| $query = $this->getLogAggregator()->queryVisitsByDimension( | ||
| array(self::USER_ID_FIELD), | ||
| "log_visit.$userIdFieldName IS NOT NULL AND log_visit.$userIdFieldName != ''", | ||
| array("LOWER(HEX($visitorIdFieldName)) as $visitorIdFieldName"), | ||
| $metrics = false, | ||
| $rankingQuery, | ||
| self::USER_ID_FIELD . ' ASC' | ||
| ); | ||
|
|
||
| if ($query === false) { | ||
| return; | ||
| } | ||
|
|
||
| $rowsCount = 0; | ||
| foreach ($query as $row) { | ||
| $rowsCount++; | ||
| $this->arrays->sumMetricsVisits($row[$userIdFieldName], $row); | ||
| $this->rememberVisitorId($row); | ||
|
|
@@ -105,8 +119,23 @@ protected function insertDayReports() | |
| { | ||
| /** @var DataTable $dataTable */ | ||
| $dataTable = $this->arrays->asDataTable(); | ||
|
|
||
| // deal w/ ranking query summary row | ||
| $rankingQuerySummaryRow = $dataTable->getRowFromLabel(DataTable::LABEL_SUMMARY_ROW); | ||
| if ($rankingQuerySummaryRow) { | ||
| $rankingQuerySummaryRowId = $dataTable->getRowIdFromLabel(DataTable::LABEL_SUMMARY_ROW); | ||
|
There was a problem hiding this comment. can you explain why this code is needed, and do we do this kind of logic in other archivers too (which ones)? if not, why is it needed in this archiver? There was a problem hiding this comment. Ranking query creates a row w/ label -1, this is added to the DataArray which is just an array, not a datatable so it has no concept of a summary row. When DataArray::asDataTable() is called, it adds the row as a regular row and not as a summary row. There was a problem hiding this comment. Ok, wondering why is this logic not needed for other plugins? There was a problem hiding this comment. It is required, Actions does it here: https://github.com/matomo-org/matomo/blob/3.x-dev/plugins/Actions/ArchivingHelper.php#L383 It's not used in Events/Contents, but that's likely a bug that wasn't dealt with since it's not tested in BlobReportLimitingTest. I'll add it to my list to fix that. There was a problem hiding this comment. @diosmosis do you mind creating an issue for this bug in case it later slips off the list? unless you will open the PR this week or so. Thanks! |
||
| $dataTable->deleteRow($rankingQuerySummaryRowId); | ||
|
|
||
| $actualSummaryRow = $dataTable->getRowFromId(DataTable::ID_SUMMARY_ROW); | ||
| if ($actualSummaryRow) { | ||
| $actualSummaryRow->sumRow($rankingQuerySummaryRow); | ||
| } else { | ||
| $dataTable->addSummaryRow($rankingQuerySummaryRow); | ||
| } | ||
| } | ||
|
|
||
| $this->setVisitorIds($dataTable); | ||
| $report = $dataTable->getSerialized($this->maximumRowsInDataTableLevelZero, null, PiwikMetrics::INDEX_NB_VISITS); | ||
| $this->getProcessor()->insertBlobRecord(self::USERID_ARCHIVE_RECORD, $report); | ||
| } | ||
|
|
||
|
|
@@ -137,4 +166,15 @@ private function setVisitorIds(DataTable $dataTable) | |
| } | ||
| } | ||
|
|
||
| private function getRankingQueryLimit() | ||
| { | ||
| $configGeneral = Config::getInstance()->General; | ||
| $configLimit = $configGeneral['archiving_ranking_query_row_limit']; | ||
| $limit = $configLimit == 0 ? 0 : max( | ||
| $configLimit, | ||
| $this->maximumRowsInDataTableLevelZero | ||
| ); | ||
| return $limit; | ||
| } | ||
|
|
||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
didn't know that worked 👍
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Neither did I :)