Leecheng/KTV
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

song 滙入大量資料問題修正 20250509

Browse Source
This commit is contained in:
allen.yan 2025-05-09 09:34:09 +08:00
parent c004ce0504
commit 8c59fced90
15 changed files with 306 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
.DS_Store vendored
View File

Binary file not shown.

BIN
app/.DS_Store vendored
View File

Binary file not shown.

90
app/Imports/ArtistDataImport.php
View File

@ -1,90 +0,0 @@
<?php
namespace App\Imports;
use App\Models\Artist;
use App\Enums\ArtistCategory;
use App\Helpers\ChineseNameConverter;
use App\Helpers\ChineseStrokesConverter;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
use Maatwebsite\Excel\Imports\HeadingRowFormatter;
class ArtistDataImport implements ToCollection, WithHeadingRow, WithChunkReading, WithBatchInserts
{
public function __construct()
{
// 關閉 heading row 格式化
HeadingRowFormatter::default('none');
}
public function collection(Collection $rows)
{
// 建立現有歌手名稱的查找表，避免重複建立
static $existingNames = null;
if ($existingNames === null) {
$existingNames = array_flip(array_map('trim', Artist::pluck('name')->all()));
}
$toInsert = [];
foreach ($rows as $row) {
$name=trim($row['歌手姓名'] ?? '');
if (empty($name)) continue;
// 若資料庫已有該名稱，跳過
if (isset($existingNames[$name])) continue;
// 字元處理
$simplified = ChineseNameConverter::convertToSimplified($name);
if (!$row->has('歌手注音')) {
$phoneticAbbr = ChineseNameConverter::getKTVZhuyinAbbr($simplified);
} else {
$phoneticAbbr = trim($row['歌手注音']);
}
$pinyinAbbr = ChineseNameConverter::getKTVPinyinAbbr($simplified);
if (!$row->has('歌手筆畫')) {
$chars = preg_split('//u', $name, -1, PREG_SPLIT_NO_EMPTY);
$firstChar = $chars[0] ?? null;
$strokesAbbr = $firstChar ? ChineseStrokesConverter::getStrokes($firstChar) : null;
} else {
$strokesAbbr = trim($row['歌手筆畫']);
}
// 準備 song 資料
$now = now();
$toInsert[] = [
'name' => $name,
'category' => ArtistCategory::tryFrom(trim($row['歌手分類'] ?? '未定義')) ?? ArtistCategory::Unset,
'simplified' => $simplified,
'phonetic_abbr' => $phoneticAbbr,
'pinyin_abbr' => $pinyinAbbr,
'strokes_abbr' => $strokesAbbr,
'enable' =>trim($row['狀態'] ?? 1),
'created_at' => $now,
'updated_at' => $now,
];
// 新增到快取，避免後面重複匯入
$existingNames[$name] = true;
}
Artist::insert($toInsert);
}
public function chunkSize(): int
{
return 100;
}
public function batchSize(): int
{
return 100;
}
public function headingRow(): int
{
return 1;
}
}

40
app/Imports/DataImport.php Normal file
View File

@ -0,0 +1,40 @@
<?php
namespace App\Imports;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Imports\HeadingRowFormatter;
use App\Jobs\ImportArtistChunkJob;
use App\Jobs\ImportSongChunkJob;
class DataImport implements ToCollection, WithHeadingRow, WithChunkReading
{
protected string $modelName;
public function __construct(string $modelName)
{
HeadingRowFormatter::default('none');
$this->modelName= $modelName;
}
public function collection(Collection $rows)
{
if($this->modelName=='Song'){
ImportSongChunkJob::dispatch($rows);
}else if($this->modelName=='Artist'){
ImportArtistChunkJob::dispatch($rows);
}else{
}
}
public function chunkSize(): int
{
return 1000;
}
public function headingRow(): int
{
return 1;
}
}

119
app/Imports/SongDataImport.php
View File

@ -1,119 +0,0 @@
<?php
namespace App\Imports;
use App\Models\Song;
use App\Models\Artist;
use App\Models\SongCategory;
use App\Enums\ArtistCategory;
use App\Enums\SongLanguageType;
use App\Enums\SongSituation;
use App\Helpers\ChineseNameConverter;
use App\Helpers\ChineseStrokesConverter;
use Illuminate\Support\Collection;
use Illuminate\Support\Str;
use Illuminate\Support\Facades\Log;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Imports\HeadingRowFormatter;
class SongDataImport implements ToCollection, WithHeadingRow, WithChunkReading
{
protected array $artistCache = [];
protected array $categoryMap = [];
public function __construct()
{
// 關閉 heading row 格式化
HeadingRowFormatter::default('none');
// 快取分類代碼對應 ID
$this->categoryMap = SongCategory::pluck('id', 'code')->toArray();
}
public function collection(Collection $rows)
{
foreach ($rows as $row) {
$songId = trim($row['編號'] ?? '');
if (!$songId) {
continue;
}
// 改為即時查詢是否已有此編號
if (Song::where('id', $songId)->exists()) {
continue;
}
// 準備 song 資料
$song = new Song([
'id' => $songId,
'name' => $songName,
'adddate' => trim($row['日期'] ?? null),
'filename' => trim($row['檔名'] ?? ''),
'language_type' => SongLanguageType::tryFrom(trim($row['語別'] ?? '')) ?? SongLanguageType::Unset,
'db_change' => trim($row['kk2'] ?? 0),//分貝增減
'vocal' => trim($row['kk6'] ?? 0),//人聲
'situation' => SongSituation::tryFrom(trim($row['kk7'] ?? '')) ?? SongSituation::Unset,//情境
'copyright01' => trim($row['版權01'] ?? ''),
'copyright02' => trim($row['版權02'] ?? ''),
'note01' => trim($row['版權03'] ?? ''),
'note02' => trim($row['版權04'] ?? ''),
'note03' => trim($row['版權05'] ?? ''),
'note04' => trim($row['版權06'] ?? ''),
'enable' => trim($row['狀態'] ?? 1),
'song_counts' => trim($row['點播次數'] ?? 0),
]);
$song->save();
// 處理關聯 - 歌手
$artistIds = [];
foreach (['歌星A', '歌星B'] as $key) {
$artistName = trim($row[$key] ?? '');
if ($artistName === '') continue;
// 若是歌星B，且與歌星A相同，則跳過
if ($key === '歌星B' && $artistName === trim($row['歌星A'] ?? '')) continue;
$artistIds[] = $this->getOrCreateArtistId($artistName);
}
$song->artists()->sync($artistIds);
// 分類處理（多個用 , 分隔）
if (!empty($row['分類'])) {
$categoryIds = [];
$codes = explode(',', $row['分類']);
foreach ($codes as $code) {
$code = trim($code);
if (isset($this->categoryMap[$code])) {
$categoryIds[] = $this->categoryMap[$code];
}
}
$song->categories()->sync($categoryIds);
}
}
}
protected function getOrCreateArtistId(string $name): int
{
if (isset($this->artistCache[$name])) {
return $this->artistCache[$name];
}
$artist = Artist::firstOrCreate(
['name' => $name],
['category' => ArtistCategory::Unset]
);
return $this->artistCache[$name] = $artist->id;
}
public function chunkSize(): int
{
return 100;
}
public function headingRow(): int
{
return 1;
}
}

52
app/Jobs/ImportArtistChunkJob.php Normal file
View File

@ -0,0 +1,52 @@
<?php
namespace App\Jobs;
use App\Models\Artist;
use App\Enums\ArtistCategory;
use Illuminate\Bus\Queueable;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Collection;
class ImportArtistChunkJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected Collection $rows;
public function __construct(Collection $rows)
{
$this->rows = $rows;
}
public function handle(): void
{
foreach ($this->rows as $index => $row) {
try {
$name = trim($row['歌手姓名'] ?? '');
if (empty($name)) {
continue;
}
if (Artist::where('name', $name)->exists()) {
continue;
}
Artist::create([
'name' => $name,
'category' => ArtistCategory::tryFrom(trim($row['歌手分類'] ?? '未定義')) ?? ArtistCategory::Unset,
'enable' => trim($row['狀態'] ?? 1),
]);
} catch (\Throwable $e) {
\Log::error("Row {$index} failed: {$e->getMessage()}", [
'row' => $row,
'trace' => $e->getTraceAsString()
]);
}
}
}
}

44
app/Jobs/ImportArtistJob.php
View File

@ -1,44 +0,0 @@
<?php
namespace App\Jobs;
use App\Imports\ArtistDataImport;
use Illuminate\Bus\Queueable;
use Illuminate\Support\Facades\Storage;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Maatwebsite\Excel\Facades\Excel;
//use Illuminate\Foundation\Queue\Queueable;
class ImportArtistJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected string $filePath;
public $timeout = 3600;
/**
* Create a new job instance.
*/
public function __construct(string $filePath)
{
$this->filePath = $filePath;
}
/**
* Execute the job.
*/
public function handle(): void
{
ini_set('memory_limit', '-1'); // ✅ 增加記憶體限制
Excel::import(new ArtistDataImport, $this->filePath);
// 匯入完成後刪除檔案
if (Storage::exists($this->filePath)) {
Storage::delete($this->filePath);
}
}
}

59
app/Jobs/ImportJob.php Normal file
View File

@ -0,0 +1,59 @@
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Support\Facades\Storage;
use Maatwebsite\Excel\Facades\Excel;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Facades\Log;
use App\Imports\DataImport;
class ImportJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected string $modelName;
protected string $filePath;
public $timeout = 3600;
public $tries = 1;
/**
* Create a new job instance.
*/
public function __construct(string $filePath,string $modelName)
{
$this->filePath = $filePath;
$this->modelName= $modelName;
}
/**
* Execute the job.
*/
public function handle(): void
{
ini_set('memory_limit', '512M'); // ✅ 增加記憶體限制
Log::info('[ImportJob] 開始處理檔案：' . $this->filePath);
try {
if (!Storage::exists($this->filePath)) {
Log::warning('[ImportJob] 檔案不存在：' . $this->filePath);
return;
}
Excel::import(new DataImport($this->modelName), $this->filePath);
Log::info('[ImportJob] 已提交所有 chunk 匯入任務。');
Storage::delete($this->filePath);
Log::info('[ImportJob] 已刪除檔案：' . $this->filePath);
} catch (\Throwable $e) {
Log::error("[ImportJob] 匯入失敗：{$e->getMessage()}", [
'file' => $this->filePath,
'trace' => $e->getTraceAsString(),
]);
}
}
}

102
app/Jobs/ImportSongChunkJob.php Normal file
View File

@ -0,0 +1,102 @@
<?php
namespace App\Jobs;
use App\Models\Song;
use App\Models\Artist;
use App\Models\SongCategory;
use App\Enums\ArtistCategory;
use App\Enums\SongLanguageType;
use App\Enums\SongSituation;
use Illuminate\Bus\Queueable;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Collection;
class ImportSongChunkJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected Collection $rows;
protected array $categoryMap = [];
public function __construct(Collection $rows)
{
$this->rows = $rows;
$this->categoryMap = SongCategory::pluck('id', 'code')->toArray();
}
public function handle(): void
{
foreach ($this->rows as $index => $row) {
$songId = trim($row['編號'] ?? '');
if (!$songId) {
continue;
}
// 改為即時查詢是否已有此編號
if (Song::where('id', $songId)->exists()) {
continue;
}
try {
// 準備 song 資料
$song = new Song([
'id' => $songId,
'name' => trim($row['歌名'] ?? ''),
'adddate' => trim($row['日期'] ?? null),
'filename' => trim($row['檔名'] ?? ''),
'language_type' => SongLanguageType::tryFrom(trim($row['語別'] ?? '')) ?? SongLanguageType::Unset,
'db_change' => trim($row['kk2'] ?? 0),//分貝增減
'vocal' => trim($row['kk6'] ?? 0),//人聲
'situation' => SongSituation::tryFrom(trim($row['kk7'] ?? '')) ?? SongSituation::Unset,//情境
'copyright01' => trim($row['版權01'] ?? ''),
'copyright02' => trim($row['版權02'] ?? ''),
'note01' => trim($row['版權03'] ?? ''),
'note02' => trim($row['版權04'] ?? ''),
'note03' => trim($row['版權05'] ?? ''),
'note04' => trim($row['版權06'] ?? ''),
'enable' => trim($row['狀態'] ?? 1),
'song_counts' => trim($row['點播次數'] ?? 0),
]);
$song->save();
// 處理關聯 - 歌手
$artistIds = [];
foreach (['歌星A', '歌星B'] as $key) {
$artistName = trim($row[$key] ?? '');
if ($artistName === '') continue;
// 若是歌星B，且與歌星A相同，則跳過
if ($key === '歌星B' && $artistName === trim($row['歌星A'] ?? '')) continue;
$artistIds[] = $this->getOrCreateArtistId($artistName);
}
$song->artists()->sync($artistIds);
// 分類處理（多個用 , 分隔）
if (!empty($row['分類'])) {
$categoryIds = [];
$codes = explode(',', $row['分類']);
foreach ($codes as $code) {
$code = trim($code);
if (isset($this->categoryMap[$code])) {
$categoryIds[] = $this->categoryMap[$code];
}
}
$song->categories()->sync($categoryIds);
}
} catch (\Throwable $e) {
\Log::error("Row {$index} failed: {$e->getMessage()}", [
'row' => $row,
'trace' => $e->getTraceAsString()
]);
}
}
}
}

57
app/Jobs/ImportSongJob.php
View File

@ -1,57 +0,0 @@
<?php
namespace App\Jobs;
use App\Imports\SongDataImport;
use Illuminate\Bus\Queueable;
use Illuminate\Support\Facades\Storage;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Maatwebsite\Excel\Facades\Excel;
//use Illuminate\Foundation\Queue\Queueable;
class ImportSongJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected string $filePath;
//public $timeout = 36000;
/**
* Create a new job instance.
*/
public function __construct(string $filePath)
{
$this->filePath = $filePath;
}
/**
* Execute the job.
*/
public function handle(): void
{
//ini_set('memory_limit', '-1'); // 無限記憶體，適合大量匯入
try {
Excel::import(new SongDataImport, $this->filePath);
// 匯入成功後再刪檔案
if (Storage::exists($this->filePath)) {
Storage::delete($this->filePath);
}
} catch (\Throwable $e) {
// 寫入錯誤日誌
\Log::error('ImportSongJob failed: ' . $e->getMessage(), [
'trace' => $e->getTraceAsString(),
'file' => $this->filePath
]);
// ❗重要：不要刪除檔案，讓失敗時可以 retry 使用同一份檔案
throw $e; // 讓 Laravel Queue 系統可以 retry
}
}
}

4
app/Livewire/Admin/ArtistImportData.php
View File

@ -8,7 +8,7 @@ use Illuminate\Support\Facades\File;
use WireUi\Traits\WireUiActions; use WireUi\Traits\WireUiActions;
use Livewire\Component; use Livewire\Component;
use Livewire\WithFileUploads; use Livewire\WithFileUploads;
use App\Jobs\ImportArtistJob; use App\Jobs\ImportJob;
class ArtistImportData extends Component class ArtistImportData extends Component
@ -55,7 +55,7 @@ class ArtistImportData extends Component
$path = $this->file->storeAs('imports', uniqid() . '_' . $this->file->getClientOriginalName()); $path = $this->file->storeAs('imports', uniqid() . '_' . $this->file->getClientOriginalName());
// 丟到 queue 執行 // 丟到 queue 執行
ImportArtistJob::dispatch($path); ImportJob::dispatch($path,'Artist');
$this->notification()->send([ $this->notification()->send([
'icon' => 'info', 'icon' => 'info',

4
app/Livewire/Admin/SongImportData.php
View File

@ -8,7 +8,7 @@ use Illuminate\Support\Facades\File;
use WireUi\Traits\WireUiActions; use WireUi\Traits\WireUiActions;
use Livewire\Component; use Livewire\Component;
use Livewire\WithFileUploads; use Livewire\WithFileUploads;
use App\Jobs\ImportSongJob; use App\Jobs\ImportJob;
class SongImportData extends Component class SongImportData extends Component
@ -55,7 +55,7 @@ class SongImportData extends Component
$path = $this->file->storeAs('imports', uniqid() . '_' . $this->file->getClientOriginalName()); $path = $this->file->storeAs('imports', uniqid() . '_' . $this->file->getClientOriginalName());
// 丟到 queue 執行 // 丟到 queue 執行
ImportSongJob::dispatch($path); ImportJob::dispatch($path,'Song');
$this->notification()->send([ $this->notification()->send([
'icon' => 'info', 'icon' => 'info',

47
database/migrations/2025_04_24_031630_create_songs_table.php Normal file
View File

@ -0,0 +1,47 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('song_library_cache', function (Blueprint $table) {
$table->integer('song_id')->primary();
$table->string('song_name', 255)->nullable();
$table->string('artistA', 255)->nullable();
$table->string('artistB', 255)->nullable();
$table->string('song_filename', 255)->nullable();
$table->string('artistA_simplified', 255)->nullable();
$table->string('ArtistB_simplified', 255)->nullable();
$table->string('artistA_category', 255)->nullable();
$table->string('artistB_category', 255)->nullable();
$table->string('song_simplified', 255)->nullable();
$table->string('situations', 255)->nullable();
$table->string('vocal', 255)->nullable();
$table->string('language_name', 50)->nullable();
$table->string('phonetic_abbr', 255)->nullable();
$table->string('artistA_phonetic', 255)->nullable();
$table->string('artistB_phonetic', 255)->nullable();
$table->string('artistA_pinyin', 255)->nullable();
$table->string('artistB_pinyin', 255)->nullable();
$table->string('pinyin_abbr', 255)->nullable();
$table->integer('song_counts')->nullable();
$table->dateTime('add_date')->nullable();
$table->dateTime('updated_at')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('song_library_cache');
}
};

BIN
storage/.DS_Store vendored
View File

Binary file not shown.

4
開發手冊.ini
View File

@ -96,8 +96,8 @@ php artisan make:observer RoomObserver --model=Room
php artisan queue:work php artisan queue:work --timeout=600 --memory=1024
php artisan queue:work --daemon --timeout=3600 --memory=5120 --tries=1 --queue=default
composer install composer install
cp .env.example .env cp .env.example .env