Skip to content

Commit f53b0a8

Browse files
naveed-ahmadnaveed-ahmad
committed
segments report
1 parent d8ee2e9 commit f53b0a8

1 file changed

Lines changed: 385 additions & 0 deletions

File tree

‎lib/tasks/segments_report.rake‎

Lines changed: 385 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,385 @@
1+
namespace :segments_report do
2+
task parse_logs: :environment do
3+
base_path = "/Volumes/Data/qul-segments/15-july"
4+
logs_path = "#{base_path}/logs/*.log"
5+
DB_FILE = "#{base_path}/segments_database.db"
6+
7+
class SegmentBase < ActiveRecord::Base
8+
self.abstract_class = true
9+
self.establish_connection(
10+
adapter: 'sqlite3',
11+
database: DB_FILE
12+
)
13+
end
14+
15+
class SegmentLog < SegmentBase
16+
self.table_name = 'logs'
17+
end
18+
19+
Dir.glob(logs_path).each do |file_path|
20+
puts "Processing #{file_path}"
21+
22+
content = File.readlines(file_path)
23+
next if content.blank?
24+
25+
filename = File.basename(file_path)
26+
reciter_id = filename[0..3].to_i
27+
surah_number = filename[4..7].to_i
28+
29+
content.each do |line|
30+
match = line.match(/^\[(.*?)\]\s*(.*)/)
31+
time = match[1]
32+
time = Time.iso8601(time)
33+
34+
if (log = match[2].to_s.strip).present?
35+
SegmentLog.create(
36+
surah_number: surah_number,
37+
reciter_id: reciter_id,
38+
timestamp: time.to_i,
39+
log: log
40+
)
41+
end
42+
end
43+
end
44+
end
45+
46+
task prepare_stats: :environment do
47+
require 'sqlite3'
48+
49+
base_path = "/Volumes/Data/qul-segments/15-july"
50+
logs_path = "#{base_path}/vs_logs/**/time-machine.json"
51+
DB_FILE = "#{base_path}/segments_data.db"
52+
53+
recitation_ids = []
54+
55+
Dir.glob(logs_path) do |file_path|
56+
session_id = file_path[%r{vs_logs/([^/]+)/}, 1]
57+
reciter_id = session_id[0..3].to_i
58+
recitation_ids << reciter_id
59+
end
60+
61+
recitations = Audio::Recitation.where(id: recitation_ids.uniq)
62+
63+
class Stats < ActiveRecord::Base
64+
self.abstract_class = true
65+
self.establish_connection(
66+
adapter: 'sqlite3',
67+
database: DB_FILE
68+
)
69+
end
70+
71+
class DetectionStat < Stats
72+
self.table_name = 'detection_stats'
73+
end
74+
75+
class Position < Stats
76+
self.table_name = 'positions'
77+
end
78+
79+
class Failure < Stats
80+
self.table_name = 'failures'
81+
end
82+
83+
class ReciterName < Stats
84+
self.table_name = 'reciters'
85+
end
86+
87+
unless Stats.connection.tables.include?("reciters")
88+
Stats.connection.create_table :reciters do |t|
89+
t.string :name
90+
t.string :audio_cdn_path
91+
t.boolean :prefix_file_name, default: false
92+
end
93+
end
94+
95+
unless Stats.connection.tables.include?("positions")
96+
Stats.connection.create_table :positions do |t|
97+
t.integer :surah_number
98+
t.integer :ayah_number
99+
t.integer :word_number
100+
t.string :word_key
101+
t.integer :reciter_id
102+
t.integer :start_time
103+
t.integer :end_time
104+
end
105+
end
106+
107+
unless Stats.connection.tables.include?("detection_stats")
108+
Stats.connection.create_table :detection_stats do |t|
109+
t.integer :surah_number
110+
t.integer :reciter_id
111+
t.string :detection_type
112+
t.integer :count
113+
end
114+
end
115+
116+
unless Stats.connection.tables.include?("failures")
117+
Stats.connection.create_table :failures do |t|
118+
t.integer :surah_number
119+
t.integer :ayah_number
120+
t.integer :word_number
121+
t.string :word_key
122+
t.string :text
123+
t.integer :reciter_id
124+
t.string :failure_type
125+
t.string :received_transcript
126+
t.string :expected_transcript
127+
t.integer :start_time
128+
t.integer :end_time
129+
end
130+
end
131+
132+
recitations.each do |recitation|
133+
resource_content = recitation.get_resource_content
134+
base_url = resource_content.meta_value("audio-cdn-url") || "https://download.quranicaudio.com"
135+
path = recitation.relative_path
136+
sample_audio_url = recitation.chapter_audio_files.where(chapter_id: 1).first.audio_url
137+
prefix_file_name = sample_audio_url.split('/').last.start_with?('00')
138+
139+
ReciterName.where(id: recitation.id).first_or_create(
140+
name: recitation.name,
141+
audio_cdn_path: "#{base_url}/#{path}",
142+
prefix_file_name: prefix_file_name
143+
)
144+
end
145+
146+
Dir.glob(logs_path) do |file_path|
147+
puts "Processing #{file_path}"
148+
149+
filename = File.basename(File.dirname(file_path))
150+
next unless filename.match?(/^\d{8}-/)
151+
152+
content = File.read(file_path)
153+
next if content.blank?
154+
155+
reciter = filename[0..3].to_i
156+
surah = filename[4..7].to_i
157+
158+
type_counts = Hash.new(0)
159+
data = JSON.parse(content)
160+
161+
data.each do |entry|
162+
type = entry["type"]
163+
type_counts[type] += 1
164+
165+
mistake = entry["mistakeWithPositions"]
166+
type_counts['FAILURE'] += 1 if mistake && type != 'FAILURE'
167+
168+
next if mistake.blank?
169+
170+
word_info = entry["position"] || entry["sessionRange"]
171+
ayah = word_info&.[]("ayahNumber") || word_info&.[]("startAyahNumber")
172+
word_num = word_info&.[]("wordNumber") || word_info&.[]("wordIndex")
173+
word_key = (ayah && word_num) ? "#{surah}:#{ayah}:#{word_num}" : "unknown"
174+
175+
Failure.create!(
176+
surah: surah,
177+
ayah: ayah,
178+
word: word_num,
179+
reciter: reciter,
180+
word_key: word_key,
181+
text: entry['word'],
182+
failure_type: mistake["mistakeType"],
183+
received_transcript: mistake["receivedTranscript"],
184+
expected_transcript: mistake["expectedTranscript"],
185+
start_time: entry["startTime"],
186+
end_time: entry["endTime"]
187+
)
188+
end
189+
190+
type_counts.each do |type, count|
191+
DetectionStat.create!(
192+
surah: surah,
193+
reciter: reciter,
194+
detection_type: type,
195+
count: count
196+
)
197+
end
198+
end
199+
200+
puts "Log parsing finished ✅ — See: #{DB_FILE}"
201+
end
202+
203+
task prepare_stats_updated: :environment do
204+
require 'sqlite3'
205+
206+
base_path = "/Volumes/Data/qul-segments/15-july"
207+
logs_path = "#{base_path}/vs_logs/**/time-machine.json"
208+
DB_FILE = "#{base_path}/segments_database.db"
209+
210+
recitation_ids = []
211+
212+
Dir.glob(logs_path) do |file_path|
213+
session_id = file_path[%r{vs_logs/([^/]+)/}, 1]
214+
reciter_id = session_id[0..3].to_i
215+
recitation_ids << reciter_id
216+
end
217+
218+
recitations = Audio::Recitation.where(id: recitation_ids.uniq)
219+
220+
class SegmentBase < ActiveRecord::Base
221+
self.abstract_class = true
222+
self.establish_connection(
223+
adapter: 'sqlite3',
224+
database: DB_FILE
225+
)
226+
end
227+
228+
class SegmentLog < SegmentBase
229+
self.table_name = 'logs'
230+
end
231+
232+
class DetectionStat < SegmentBase
233+
self.table_name = 'detection_stats'
234+
end
235+
236+
class Position < SegmentBase
237+
self.table_name = 'positions'
238+
end
239+
240+
class Failure < SegmentBase
241+
self.table_name = 'failures'
242+
end
243+
244+
class ReciterName < SegmentBase
245+
self.table_name = 'reciters'
246+
end
247+
248+
unless SegmentBase.connection.tables.include?("reciters")
249+
SegmentBase.connection.create_table :reciters do |t|
250+
t.string :name
251+
t.string :audio_cdn_path
252+
t.boolean :prefix_file_name, default: false
253+
end
254+
end
255+
256+
unless SegmentBase.connection.tables.include?("positions")
257+
SegmentBase.connection.create_table :positions do |t|
258+
t.integer :surah_number
259+
t.integer :ayah_number
260+
t.integer :word_number
261+
t.string :word_key
262+
t.integer :reciter_id
263+
t.integer :start_time
264+
t.integer :end_time
265+
end
266+
end
267+
268+
unless SegmentBase.connection.tables.include?("detection_stats")
269+
SegmentBase.connection.create_table :detection_stats do |t|
270+
t.integer :surah_number
271+
t.integer :ayah_number
272+
t.integer :reciter_id
273+
t.string :detection_type
274+
t.integer :count
275+
end
276+
end
277+
278+
unless SegmentBase.connection.tables.include?("logs")
279+
SegmentBase.connection.create_table :logs do |t|
280+
t.integer :surah_number
281+
t.integer :reciter_id
282+
t.integer :timestamp
283+
t.string :log
284+
end
285+
end
286+
287+
unless SegmentBase.connection.tables.include?("failures")
288+
SegmentBase.connection.create_table :failures do |t|
289+
t.integer :surah_number
290+
t.integer :ayah_number
291+
t.integer :word_number
292+
t.string :word_key
293+
t.string :text
294+
t.integer :reciter_id
295+
t.string :failure_type
296+
t.string :received_transcript
297+
t.string :expected_transcript
298+
t.integer :start_time
299+
t.integer :end_time
300+
end
301+
end
302+
303+
recitations.each do |recitation|
304+
resource_content = recitation.get_resource_content
305+
base_url = resource_content.meta_value("audio-cdn-url") || "https://download.quranicaudio.com"
306+
path = recitation.relative_path
307+
sample_audio_url = recitation.chapter_audio_files.where(chapter_id: 1).first.audio_url
308+
prefix_file_name = sample_audio_url.split('/').last.start_with?('00')
309+
310+
ReciterName.where(id: recitation.id).first_or_create(
311+
name: recitation.name,
312+
audio_cdn_path: "#{base_url}/#{path}",
313+
prefix_file_name: prefix_file_name
314+
)
315+
end
316+
317+
Dir.glob(logs_path) do |file_path|
318+
puts "Processing #{file_path}"
319+
320+
filename = File.basename(File.dirname(file_path))
321+
next unless filename.match?(/^\d{8}-/)
322+
323+
content = File.read(file_path)
324+
next if content.blank?
325+
326+
reciter_id = filename[0..3].to_i
327+
surah_number = filename[4..7].to_i
328+
329+
type_counts = Hash.new(0)
330+
data = JSON.parse(content)
331+
332+
data.each do |entry|
333+
type = entry["type"]
334+
type_counts[type] += 1
335+
336+
mistake = entry["mistakeWithPositions"]
337+
type_counts['FAILURE'] += 1 if mistake && type != 'FAILURE'
338+
339+
word_info = entry["position"] || entry["sessionRange"]
340+
ayah_number = word_info&.[]("ayahNumber") || word_info&.[]("startAyahNumber")
341+
word_number = word_info&.[]("wordNumber") || word_info&.[]("wordIndex")
342+
word_key = (ayah_number && word_number) ? "#{surah_number}:#{ayah_number}:#{word_number}" : "unknown"
343+
344+
if mistake.present?
345+
Failure.create!(
346+
surah_number: surah_number,
347+
ayah_number: ayah_number,
348+
word_number: word_number,
349+
reciter_id: reciter_id,
350+
word_key: word_key,
351+
text: entry['word'],
352+
failure_type: mistake["mistakeType"],
353+
received_transcript: mistake["receivedTranscript"],
354+
expected_transcript: mistake["expectedTranscript"],
355+
start_time: entry["startTime"],
356+
end_time: entry["endTime"]
357+
)
358+
end
359+
360+
if ayah_number && word_number && entry["startTime"] && entry["endTime"]
361+
Position.create!(
362+
surah_number: surah_number,
363+
ayah_number: ayah_number,
364+
word_number: word_number,
365+
word_key: word_key,
366+
reciter_id: reciter_id,
367+
start_time: entry["startTime"],
368+
end_time: entry["endTime"]
369+
)
370+
end
371+
end
372+
373+
type_counts.each do |type, count|
374+
DetectionStat.create!(
375+
surah_number: surah_number,
376+
reciter_id: reciter_id,
377+
detection_type: type,
378+
count: count
379+
)
380+
end
381+
end
382+
383+
puts "Log parsing finished, see the result DB #{DB_FILE}"
384+
end
385+
end

0 commit comments

Comments
 (0)