first commit
Some checks failed
Build / run (push) Has been cancelled

This commit is contained in:
maher
2025-10-29 11:42:25 +01:00
commit 703f50a09d
4595 changed files with 385164 additions and 0 deletions

View File

@@ -0,0 +1,260 @@
<?php
namespace Common\Database\Datasource;
use Common\Database\Datasource\Filters\AlgoliaFilterer;
use Common\Database\Datasource\Filters\ElasticFilterer;
use Common\Database\Datasource\Filters\MeilisearchFilterer;
use Common\Database\Datasource\Filters\MysqlFilterer;
use Common\Database\Datasource\Filters\TntFilterer;
use Illuminate\Database\Eloquent\Builder as EloquentBuilder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\Relation;
use Illuminate\Database\Query\Builder as QueryBuilder;
use Illuminate\Database\Query\Expression;
use Illuminate\Pagination\AbstractPaginator;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Str;
use Laravel\Scout\Builder as ScoutBuilder;
use Laravel\Scout\Searchable;
use Matchish\ScoutElasticSearch\Engines\ElasticSearchEngine;
use const App\Providers\WORKSPACED_RESOURCES;
class Datasource
{
protected EloquentBuilder|Relation $builder;
protected Model $model;
protected array $params;
protected bool $queryBuilt = false;
public DatasourceFilters $filters;
public array|null|false $order = null;
// sometimes we might need to order by ID or something else, if primary column
// is not guaranteed to be unique, to avoid duplicated items in pagination
public string|null $secondaryOrderCol = null;
protected ?ScoutBuilder $scoutBuilder;
public function __construct(
$model,
array $params,
DatasourceFilters $filters = null,
protected string $filtererName = 'mysql',
) {
$this->model = $model->getModel();
$this->params = $this->toCamelCase($params);
$this->builder = $model instanceof Model ? $model->newQuery() : $model;
$this->filters =
$filters ?? new DatasourceFilters($this->params['filters'] ?? null);
}
public function paginate(): AbstractPaginator
{
$this->buildQuery();
$perPage = $this->limit();
$page = (int) $this->param('page', 1);
$columns = empty($this->builder->getQuery()->columns)
? ['*']
: $this->builder->getQuery()->columns;
if ($this->param('paginate') === 'lengthAware') {
return $this->scoutBuilder instanceof ScoutBuilder
? $this->scoutBuilder->paginate($perPage, 'page', $page)
: $this->builder->paginate($perPage, $columns, 'page', $page);
} else {
return $this->scoutBuilder instanceof ScoutBuilder
? $this->scoutBuilder->simplePaginate($perPage, 'page', $page)
: $this->builder->simplePaginate(
$perPage,
$columns,
'page',
$page,
);
}
}
public function get(): Collection
{
$this->buildQuery();
return $this->builder->limit($this->limit())->get();
}
public function param(string $name, $default = null)
{
return Arr::get($this->params, Str::camel($name)) ?: $default;
}
public function buildQuery(): self
{
if ($this->queryBuilt) {
return $this;
}
$with = array_filter(explode(',', $this->param('with', '')));
$withCount = array_filter(explode(',', $this->param('withCount', '')));
$searchTerm = $this->param('query');
// load specified relations and counts
if (!empty($with)) {
$this->builder->with($with);
}
if (!empty($withCount)) {
$this->builder->withCount($withCount);
}
$this->applyWorkspaceFilter();
$filterer = $this->resolveFilterer($searchTerm);
$this->scoutBuilder = (new $filterer(
$this->builder,
$this->filters,
$searchTerm,
))->apply();
// allow caller class to override order or
// prevent it completely by setting "false"
if ($this->order !== false) {
$order = $this->getOrder();
if (isset($order['col'])) {
$orderCol = str_replace(
$this->builder->getModel()->getTable() . '.',
'',
$order['col'],
);
$methodName = Str::camel('orderBy' . ucfirst($orderCol));
$scopeMethodName = 'scope' . ucfirst($methodName);
if (
method_exists($this->builder->getModel(), $methodName) ||
method_exists($this->builder->getModel(), $scopeMethodName)
) {
$this->builder->$methodName($order['dir']);
} else {
$this->builder->orderBy(
Str::snake($order['col']),
$order['dir'] ?? 'desc',
);
}
if ($this->secondaryOrderCol) {
$this->builder->orderBy(
$this->secondaryOrderCol,
$order['dir'] ?? 'desc',
);
}
}
}
$this->queryBuilt = true;
return $this;
}
private function resolveFilterer(string $searchTerm = null): string
{
$filtererName = $this->filtererName;
if (
!$searchTerm ||
!in_array(Searchable::class, class_uses_recursive($this->model)) ||
$filtererName === 'mysql'
) {
return MysqlFilterer::class;
} elseif ($filtererName === 'meilisearch') {
return MeilisearchFilterer::class;
} elseif ($filtererName === 'tntsearch') {
return TntFilterer::class;
} elseif ($filtererName === 'algolia') {
return AlgoliaFilterer::class;
} elseif ($filtererName === ElasticSearchEngine::class) {
return ElasticFilterer::class;
}
return MysqlFilterer::class;
}
private function applyWorkspaceFilter(): void
{
if (
!config('common.site.workspaces_integrated') ||
!in_array(get_class($this->model), WORKSPACED_RESOURCES)
) {
return;
}
if ($workspaceId = $this->param('workspaceId')) {
$this->filters->where('workspace_id', '=', $workspaceId);
} elseif ($userId = $this->param('userId')) {
$this->filters
->where('user_id', '=', $userId)
->where('workspace_id', '=', 0);
}
}
public function getOrder(
string $defaultOrderCol = 'updated_at',
string $defaultOrderDir = 'desc',
): array {
if (isset($this->order['col'])) {
$orderCol = $this->order['col'];
$orderDir = $this->order['dir'];
// order might be a single string: "column|direction"
} elseif ($specifiedOrder = $this->param('order')) {
$parts = preg_split('(\||:)', $specifiedOrder);
$orderCol = Arr::get($parts, 0, $defaultOrderCol);
$orderDir = Arr::get($parts, 1, $defaultOrderDir);
// order might be as separate params
} elseif ($this->param('orderBy') || $this->param('orderDir')) {
$orderCol = $this->param('orderBy');
$orderDir = $this->param('orderDir');
// try ordering be relevance, if it's a search query and
// using mysql fulltext, finally default to "updated_at" column
} elseif ($this->hasRelevanceColumn()) {
$orderCol = 'relevance';
$orderDir = 'desc';
} else {
$orderCol = $defaultOrderCol;
$orderDir = $defaultOrderDir;
}
if ($orderCol !== 'relevance' && !Str::endsWith($orderCol, '_count')) {
$orderCol = $this->builder->qualifyColumn($orderCol);
}
return [
'col' => $orderCol,
'dir' => $orderDir,
];
}
private function toCamelCase(array $params): array
{
return collect($params)
->keyBy(function ($value, $key) {
return Str::camel($key);
})
->toArray();
}
private function hasRelevanceColumn(): bool
{
return !!Arr::first($this->getQueryBuilder() ?? [], function ($col) {
return $col instanceof Expression &&
Str::endsWith($col->getValue(), 'AS relevance');
});
}
private function limit(): int
{
if ($this->param('perPage')) {
return (int) $this->param('perPage');
} else {
return $this->getQueryBuilder()->limit ?? 15;
}
}
private function getQueryBuilder(): QueryBuilder
{
$query = $this->builder->getQuery();
if ($query instanceof EloquentBuilder) {
$query = $query->getQuery();
}
return $query;
}
}

View File

@@ -0,0 +1,131 @@
<?php
namespace Common\Database\Datasource;
use Auth;
use Illuminate\Support\Collection;
class DatasourceFilters
{
private array $filters;
public function __construct(?string $encodedFilters = '')
{
$this->filters = $this->decodeFilters($encodedFilters);
}
public function getAll(): array
{
return $this->filters;
}
public function empty(): bool
{
return empty($this->filters);
}
public function has(string $key): bool {
foreach ($this->filters as $filter) {
if ($filter['key'] === $key) {
return true;
}
}
return false;
}
public function where(string $key, string $operator, $value): self
{
if ($value instanceof Collection) {
$value = $value->toArray();
}
$this->filters[] = [
'key' => $key,
'operator' => $operator,
'value' => $value,
];
return $this;
}
public function getAndRemove(
string $key,
string $operator = null,
$value = null,
): ?array {
// use func_get_args as "null" is a valid value, need
// to check whether if it was actually passed by user
$args = func_get_args();
foreach ($this->filters as $key => $filter) {
if (
$filter['key'] === $args[0] &&
(!isset($args[1]) || $filter['operator'] === $args[1]) &&
(!isset($args[2]) || $filter['value'] === $args[2])
) {
unset($this->filters[$key]);
return $filter;
}
}
return null;
}
private function decodeFilters(?string $filterString): array
{
if ($filterString) {
$filters = json_decode(
base64_decode(urldecode($filterString)),
true,
);
return collect($filters)
->flatMap(fn($filter) => $this->normalizeFilter($filter))
->filter()
->toArray();
} else {
return [];
}
}
private function normalizeFilter(array $filter): ?array
{
$value = $filter['value'] ?? null;
$operator = $filter['operator'] ?? '=';
$value = $this->replaceValuePlaceholders($value);
if (is_array($value)) {
// filtering by normalized model
if (isset($value['id'])) {
$value = $this->replaceValuePlaceholders($value['id']);
// "value" contains both value and operator
} elseif (array_key_exists('value', $value)) {
$operator = $value['operator'] ?? $operator;
$value = $this->replaceValuePlaceholders($value['value']);
}
}
$filters = [
// preserve any extra keys that might be present
array_merge($filter, [
'key' => $filter['key'],
'operator' => $operator,
'value' => $value,
]),
];
// filter might have some extra static filters, for example to restrict by model type
if (isset($filter['extraFilters'])) {
$filters[] = $filter['extraFilters'];
}
return $filters;
}
private function replaceValuePlaceholders($value)
{
if ($value === '{authId}') {
return Auth::id();
}
return $value;
}
}

View File

@@ -0,0 +1,54 @@
<?php
namespace Common\Database\Datasource\Filters;
use Algolia\AlgoliaSearch\SearchIndex;
use Common\Database\Datasource\Filters\Traits\NormalizesFiltersForFulltextEngines;
use Illuminate\Support\Str;
use Laravel\Scout\Builder;
class AlgoliaFilterer extends BaseFilterer
{
use NormalizesFiltersForFulltextEngines;
public function apply(): ?Builder
{
return $this->query
->getModel()
->search($this->searchTerm, function (
SearchIndex $algolia,
string $query,
array $options
) {
$filters = $this->prepareFiltersForAlgolia();
$filters = implode(' AND ', $filters);
if ($filters) {
$options['filters'] = $filters;
}
return $algolia->search($query, $options);
});
}
private function prepareFiltersForAlgolia(): array
{
$filters = $this->normalizeFilters($this->filters->getAll());
return array_map(function ($filter) {
$prefix = '';
if (Str::contains($filter['operator'], '!')) {
$filter['operator'] = str_replace('!', '', $filter['operator']);
$prefix = 'NOT ';
}
if (!is_numeric($filter['value']) && $filter['operator'] === '=') {
$filter['operator'] = ':';
}
if (is_array($filter['value'])) {
$filter['value'] = implode(',', $filter['value']);
}
return $prefix . implode('', $filter);
}, $filters);
}
}

View File

@@ -0,0 +1,18 @@
<?php
namespace Common\Database\Datasource\Filters;
use Common\Database\Datasource\DatasourceFilters;
use Laravel\Scout\Builder as ScoutBuilder;
abstract class BaseFilterer
{
public function __construct(
protected $query,
protected DatasourceFilters $filters,
protected string|null $searchTerm = null
) {
}
abstract public function apply(): ?ScoutBuilder;
}

View File

@@ -0,0 +1,47 @@
<?php
namespace Common\Database\Datasource\Filters;
use Common\Database\Datasource\Filters\Traits\NormalizesFiltersForFulltextEngines;
use Elasticsearch\Client;
use Laravel\Scout\Builder;
use Matchish\ScoutElasticSearch\ElasticSearch\Params\Search as SearchParams;
use ONGR\ElasticsearchDSL\Query\TermLevel\TermQuery;
use ONGR\ElasticsearchDSL\Query\TermLevel\TermsQuery;
use ONGR\ElasticsearchDSL\Search;
class ElasticFilterer extends BaseFilterer
{
use NormalizesFiltersForFulltextEngines;
public function apply(): ?Builder
{
return $this->query
->getModel()
->search($this->searchTerm, function (
Client $client,
Search $body
) {
$filters = $this->normalizeFilters($this->filters->getAll());
foreach ($filters as $filter) {
if (is_array($filter['value'])) {
$filter = new TermsQuery(
$filter['key'],
$filter['value'],
);
} else {
$filter = new TermQuery(
$filter['key'],
$filter['value'],
);
}
$body->addPostFilter($filter);
}
$params = new SearchParams(
$this->query->getModel()->searchableAs(),
$body->toArray(),
);
return $client->search($params->toArray());
});
}
}

View File

@@ -0,0 +1,61 @@
<?php
namespace Common\Database\Datasource\Filters;
use Common\Database\Datasource\Filters\Traits\NormalizesFiltersForFulltextEngines;
use Laravel\Scout\Builder;
class MeilisearchFilterer extends BaseFilterer
{
use NormalizesFiltersForFulltextEngines;
public function apply(): Builder
{
return $this->query
->getModel()
->search($this->searchTerm, function (
$driver,
?string $query,
array $options
) {
$filters = $this->prepareFiltersForMeilisearch();
$filters = implode(' AND ', $filters);
if ($filters) {
$options['filter'] = $filters;
}
return $driver->search($query, $options);
});
}
private function prepareFiltersForMeilisearch(): array
{
$filters = $this->normalizeFilters($this->filters->getAll());
return array_map(function ($filter) {
if (is_array($filter['value'])) {
$values = array_map(
fn($v) => $this->createFilterString(
$filter['key'],
$filter['operator'],
$v,
),
$filter['value'],
);
return '(' . implode(' OR ', $values) . ')';
} else {
return $this->createFilterString(
$filter['key'],
$filter['operator'],
$filter['value'],
);
}
}, $filters);
}
private function createFilterString(
string $key,
string $operator,
$value
): string {
return "$key $operator $value";
}
}

View File

@@ -0,0 +1,22 @@
<?php
namespace Common\Database\Datasource\Filters;
use Common\Database\Datasource\Filters\Traits\SupportsMysqlFilters;
use Laravel\Scout\Builder;
class MysqlFilterer extends BaseFilterer
{
use SupportsMysqlFilters;
public function apply(): ?Builder
{
$this->applyMysqlFilters($this->filters, $this->query);
if ($this->searchTerm) {
$this->query->mysqlSearch($this->searchTerm);
}
return null;
}
}

View File

@@ -0,0 +1,24 @@
<?php
namespace Common\Database\Datasource\Filters;
use Common\Database\Datasource\Filters\Traits\SupportsMysqlFilters;
use Laravel\Scout\Builder;
class TntFilterer extends BaseFilterer
{
use SupportsMysqlFilters;
public function apply(): ?Builder
{
$constrains = $this->applyMysqlFilters(
$this->filters,
$this->query->getModel()->newInstance(),
);
return $this->query
->getModel()
->search($this->searchTerm)
->constrain($constrains);
}
}

View File

@@ -0,0 +1,75 @@
<?php
namespace Common\Database\Datasource\Filters\Traits;
use Carbon\Carbon;
use Illuminate\Support\Arr;
use Illuminate\Support\Str;
trait NormalizesFiltersForFulltextEngines
{
protected function normalizeFilters(array $filters): array
{
$normalizedFilters = [];
foreach ($filters as $index => $filter) {
// flatten "between" filter
if ($filter['operator'] === 'between') {
if ($start = Arr::get($filter, 'value.start')) {
$normalizedFilters[] = [
'key' => $filter['key'],
'operator' => '>=',
'value' => Carbon::parse($start)->timestamp,
];
}
if ($end = Arr::get($filter, 'value.end')) {
$normalizedFilters[] = [
'key' => $filter['key'],
'operator' => '<=',
'value' => Carbon::parse($end)->timestamp,
];
}
} else {
// normalize value and operator, so it's accepted by meilisearch, elastic and algolia
$normalizedFilters[] = [
'key' => $filter['key'],
'operator' => $this->normalizeFilterOperator($filter),
'value' => $this->normalizeFilterValue($filter),
];
}
}
return $normalizedFilters;
}
protected function normalizeFilterValue(array $filter): array|string
{
$value = $filter['value'];
if (is_string($value) && Str::contains($value, ' ')) {
return "'$value'";
} elseif ($value === null) {
return '_null';
} elseif ($value === false) {
return 'false';
} elseif ($value === true) {
return 'true';
} elseif (
in_array($filter['key'], $this->query->getModel()->getDates())
) {
return Carbon::parse($value)->timestamp;
} else {
return $value;
}
}
protected function normalizeFilterOperator(array $filter): string
{
if ($filter['operator'] === 'has') {
return '=';
} elseif ($filter['operator'] === 'doesntHave') {
return '!=';
} else {
return $filter['operator'];
}
}
}

View File

@@ -0,0 +1,180 @@
<?php
namespace Common\Database\Datasource\Filters\Traits;
use Common\Database\Datasource\DatasourceFilters;
use Illuminate\Database\Eloquent\Relations\BelongsToMany;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
use Illuminate\Database\Query\Builder;
use Illuminate\Support\Facades\DB;
trait SupportsMysqlFilters
{
public function applyMysqlFilters(DatasourceFilters $filters, $query)
{
foreach ($filters->getAll() as $filter) {
if ($filter['value'] === 'null') {
$filter['value'] = null;
} elseif ($filter['value'] === 'false') {
$filter['value'] = false;
}
if ($filter['value'] === 'true') {
$filter['value'] = true;
}
if ($filter['operator'] === 'between') {
$query->whereBetween(
$query->getModel()->qualifyColumn($filter['key']),
[$filter['value']['start'], $filter['value']['end']],
);
} elseif (
$filter['operator'] === 'has' ||
$filter['operator'] === 'doesntHave'
) {
$relName = $filter['key'];
$relation = $query->getModel()->$relName();
if (
$relation instanceof HasMany ||
$relation instanceof HasOne
) {
$query = $this->filterByHasManyRelation(
$query,
$relation,
$filter,
);
} elseif ($relation instanceof BelongsToMany) {
$query = $this->filterByManyToManyRelation(
$query,
$relation,
$filter,
);
}
} elseif ($filter['operator'] === 'hasAll') {
$genreIds = $filter['value'];
$relName = $filter['key'];
$relation = $query->getModel()->$relName();
$query->whereIn(
$query->getModel()->qualifyColumn('id'),
fn(Builder $query) => $query
->select($relation->getQualifiedForeignPivotKeyName())
->from($relation->getTable())
->whereIn(
$relation->getQualifiedRelatedPivotKeyName(),
$genreIds,
)
->when(
count($genreIds) > 1,
fn(Builder $query) => $query
->groupBy(
$relation->getQualifiedForeignPivotKeyName(),
)
->having(
DB::raw('COUNT(*)'),
'=',
count($genreIds),
),
),
);
} elseif (
$query->hasNamedScope('where' . ucfirst($filter['key']))
) {
$query->{'where' . ucfirst($filter['key'])}(
$filter['value'],
$filter['operator'],
$filter['key'],
);
} else {
$query = $query->where(
$query->qualifyColumn($filter['key']),
$filter['operator'],
$filter['value'],
);
}
}
return $query;
}
private function filterByHasManyRelation($query, $relation, array $filter)
{
$related = $relation->getRelated()->getTable();
$foreignKey = $relation->getQualifiedForeignKeyName();
$parentKey = $relation->getQualifiedParentKeyName();
// use left join to check if model has any of specified relations
if ($filter['value'] === '*') {
$query
// prevent null values from being returned when using left join
->when(empty($query->getQuery()->getColumns()), function ($q) {
$q->select($q->getModel()->getTable() . '.*');
})
->leftJoin($related, $foreignKey, '=', $parentKey)
->where(
$foreignKey,
$filter['operator'] === 'doesntHave' ? '=' : '!=',
null,
);
// use left join to check if model has relation with specified ID
} else {
$query
->leftJoin($related, $foreignKey, '=', $parentKey)
->where(
"$related.id",
$filter['operator'] === 'has' ? '=' : '!=',
$filter['value'],
);
if ($filter['operator'] === 'doesntHave') {
$this->query->orWhere("$related.id", null);
}
}
return $query;
}
private function filterByManyToManyRelation(
$query,
$relation,
array $filter,
) {
if ($filter['operator'] === 'has') {
$values = is_array($filter['value'])
? $filter['value']
: [$filter['value']];
$query->join(
$relation->getTable(),
$relation->getQualifiedParentKeyName(),
'=',
$relation->getQualifiedForeignPivotKeyName(),
);
$query->where(function ($q) use ($values, $relation) {
foreach ($values as $value) {
$q->orWhere(
$relation->getQualifiedRelatedPivotKeyName(),
'=',
$value,
);
}
});
} elseif ($filter['operator'] === 'doesntHave') {
$table = $query->getModel()->getTable();
$query->whereNotIn("$table.id", function (Builder $builder) use (
$filter,
$query,
) {
$relName = $filter['key'];
$relation = $query->getModel()->$relName();
$builder
->select($relation->getQualifiedForeignPivotKeyName())
->from($relation->getTable())
->where(
$relation->getQualifiedRelatedPivotKeyName(),
$filter['value'],
);
});
}
return $query;
}
}