Merge branch 'bug_621_caid_info'
[vuplus_dvbapp] / lib / dvb / tstools.cpp
1 #define _ISOC99_SOURCE /* for llabs */
2 #include <lib/dvb/tstools.h>
3 #include <lib/base/eerror.h>
4 #include <unistd.h>
5 #include <fcntl.h>
6
7 #include <stdio.h>
8
9 eDVBTSTools::eDVBTSTools()
10 {
11         m_pid = -1;
12         m_maxrange = 256*1024;
13         
14         m_begin_valid = 0;
15         m_end_valid = 0;
16         
17         m_use_streaminfo = 0;
18         m_samples_taken = 0;
19         
20         m_last_filelength = 0;
21         
22         m_futile = 0;
23 }
24
25 void eDVBTSTools::closeSource()
26 {
27         m_source = NULL;
28 }
29
30 eDVBTSTools::~eDVBTSTools()
31 {
32         closeSource();
33 }
34
35 int eDVBTSTools::openFile(const char *filename, int nostreaminfo)
36 {
37         eRawFile *f = new eRawFile();
38         ePtr<iTsSource> src = f;
39
40         if (f->open(filename, 1) < 0)
41                 return -1;
42
43         setSource(src, nostreaminfo ? NULL : filename);
44
45         return 0;
46 }
47
48 void eDVBTSTools::setSource(ePtr<iTsSource> &source, const char *stream_info_filename)
49 {
50         closeFile();
51
52         m_source = source;
53
54         if (stream_info_filename)
55         {
56                 eDebug("loading streaminfo for %s", stream_info_filename);
57                 m_streaminfo.load(stream_info_filename);
58         }
59         
60         if (!m_streaminfo.empty())
61                 m_use_streaminfo = 1;
62         else
63         {
64 //              eDebug("no recorded stream information available");
65                 m_use_streaminfo = 0;
66         }
67
68         m_samples_taken = 0;
69 }
70
71 void eDVBTSTools::closeFile()
72 {
73         if (m_source)
74                 closeSource();
75 }
76
77 void eDVBTSTools::setSyncPID(int pid)
78 {
79         m_pid = pid;
80 }
81
82 void eDVBTSTools::setSearchRange(int maxrange)
83 {
84         m_maxrange = maxrange;
85 }
86
87         /* getPTS extracts a pts value from any PID at a given offset. */
88 int eDVBTSTools::getPTS(off_t &offset, pts_t &pts, int fixed)
89 {
90         if (m_use_streaminfo)
91                 if (!m_streaminfo.getPTS(offset, pts))
92                         return 0;
93         
94         if (!m_source || !m_source->valid())
95                 return -1;
96
97         offset -= offset % 188;
98
99         int left = m_maxrange;
100         
101         while (left >= 188)
102         {
103                 unsigned char packet[188];
104                 if (m_source->read(offset, packet, 188) != 188)
105                 {
106                         eDebug("read error");
107                         break;
108                 }
109                 left -= 188;
110                 offset += 188;
111
112                 if (packet[0] != 0x47)
113                 {
114                         eDebug("resync");
115                         int i = 0;
116                         while (i < 188)
117                         {
118                                 if (packet[i] == 0x47)
119                                         break;
120                                 ++i;
121                                 --offset;
122                         }
123                         continue;
124                 }
125                 
126                 int pid = ((packet[1] << 8) | packet[2]) & 0x1FFF;
127                 int pusi = !!(packet[1] & 0x40);
128                 
129 //              printf("PID %04x, PUSI %d\n", pid, pusi);
130
131                 unsigned char *payload;
132                 
133                         /* check for adaption field */
134                 if (packet[3] & 0x20)
135                 {
136                         if (packet[4] >= 183)
137                                 continue;
138                         if (packet[4])
139                         {
140                                 if (packet[5] & 0x10) /* PCR present */
141                                 {
142                                         pts  = ((unsigned long long)(packet[ 6]&0xFF)) << 25;
143                                         pts |= ((unsigned long long)(packet[ 7]&0xFF)) << 17;
144                                         pts |= ((unsigned long long)(packet[ 8]&0xFE)) << 9;
145                                         pts |= ((unsigned long long)(packet[ 9]&0xFF)) << 1;
146                                         pts |= ((unsigned long long)(packet[10]&0x80)) >> 7;
147                                         offset -= 188;
148                                         eDebug("PCR %16llx found at %lld pid %02x (%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x)", pts, offset, pid, packet[0], packet[1], packet[2], packet[3], packet[4], packet[5], packet[6], packet[7], packet[8], packet[9], packet[10]);
149                                         if (fixed && fixupPTS(offset, pts))
150                                                 return -1;
151                                         return 0;
152                                 }
153                         }
154                         payload = packet + packet[4] + 4 + 1;
155                 } else
156                         payload = packet + 4;
157
158 /*              if (m_pid >= 0)
159                         if (pid != m_pid)
160                                 continue; */
161                 if (!pusi)
162                         continue;
163
164                         /* somehow not a startcode. (this is invalid, since pusi was set.) ignore it. */
165                 if (payload[0] || payload[1] || (payload[2] != 1))
166                         continue;
167
168                 if (payload[3] == 0xFD)
169                 { // stream use extension mechanism defined in ISO 13818-1 Amendment 2
170                         if (payload[7] & 1) // PES extension flag
171                         {
172                                 int offs = 0;
173                                 if (payload[7] & 0x80) // pts avail
174                                         offs += 5;
175                                 if (payload[7] & 0x40) // dts avail
176                                         offs += 5;
177                                 if (payload[7] & 0x20) // escr avail
178                                         offs += 6;
179                                 if (payload[7] & 0x10) // es rate
180                                         offs += 3;
181                                 if (payload[7] & 0x8) // dsm trickmode
182                                         offs += 1;
183                                 if (payload[7] & 0x4) // additional copy info
184                                         offs += 1;
185                                 if (payload[7] & 0x2) // crc
186                                         offs += 2;
187                                 if (payload[8] < offs)
188                                         continue;
189                                 uint8_t pef = payload[9+offs++]; // pes extension field
190                                 if (pef & 1) // pes extension flag 2
191                                 {
192                                         if (pef & 0x80) // private data flag
193                                                 offs += 16;
194                                         if (pef & 0x40) // pack header field flag
195                                                 offs += 1;
196                                         if (pef & 0x20) // program packet sequence counter flag
197                                                 offs += 2;
198                                         if (pef & 0x10) // P-STD buffer flag
199                                                 offs += 2;
200                                         if (payload[8] < offs)
201                                                 continue;
202                                         uint8_t stream_id_extension_len = payload[9+offs++] & 0x7F;
203                                         if (stream_id_extension_len >= 1)
204                                         {
205                                                 if (payload[8] < (offs + stream_id_extension_len) )
206                                                         continue;
207                                                 if (payload[9+offs] & 0x80) // stream_id_extension_bit (should not set)
208                                                         continue;
209                                                 switch (payload[9+offs])
210                                                 {
211                                                 case 0x55 ... 0x5f: // VC-1
212                                                         break;
213                                                 case 0x71: // AC3 / DTS
214                                                         break;
215                                                 default:
216                                                         eDebug("skip unknwn stream_id_extension %02x\n", payload[9+offs]);
217                                                         continue;
218                                                 }
219                                         }
220                                         else
221                                                 continue;
222                                 }
223                                 else
224                                         continue;
225                         }
226                         else
227                                 continue;
228                 }
229                         /* drop non-audio, non-video packets because other streams
230                            can be non-compliant.*/
231                 else if (((payload[3] & 0xE0) != 0xC0) &&  // audio
232                         ((payload[3] & 0xF0) != 0xE0)) // video
233                         continue;
234
235                 if (payload[7] & 0x80) /* PTS */
236                 {
237                         pts  = ((unsigned long long)(payload[ 9]&0xE))  << 29;
238                         pts |= ((unsigned long long)(payload[10]&0xFF)) << 22;
239                         pts |= ((unsigned long long)(payload[11]&0xFE)) << 14;
240                         pts |= ((unsigned long long)(payload[12]&0xFF)) << 7;
241                         pts |= ((unsigned long long)(payload[13]&0xFE)) >> 1;
242                         offset -= 188;
243
244                         eDebug("PTS %16llx found at %lld pid %02x stream: %02x", pts, offset, pid, payload[3]);
245
246                                 /* convert to zero-based */
247                         if (fixed && fixupPTS(offset, pts))
248                                 return -1;
249                         return 0;
250                 }
251         }
252         
253         return -1;
254 }
255
256 int eDVBTSTools::fixupPTS(const off_t &offset, pts_t &now)
257 {
258         if (m_use_streaminfo)
259         {
260                 if (!m_streaminfo.fixupPTS(offset, now))
261                         return 0;
262         } else
263         {
264                         /* for the simple case, we assume one epoch, with up to one wrap around in the middle. */
265                 calcBegin();
266                 if (!m_begin_valid)
267                 {       
268                         eDebug("begin not valid, can't fixup");
269                         return -1;
270                 }
271                 
272                 pts_t pos = m_pts_begin;
273                 if ((now < pos) && ((pos - now) < 90000 * 10))
274                 {       
275                         pos = 0;
276                         return 0;
277                 }
278                 
279                 if (now < pos) /* wrap around */
280                         now = now + 0x200000000LL - pos;
281                 else
282                         now -= pos;
283                 return 0;
284         }
285         eDebug("eDVBTSTools::fixupPTS failed!");
286         return -1;
287 }
288
289 int eDVBTSTools::getOffset(off_t &offset, pts_t &pts, int marg)
290 {
291         eDebug("getOffset for pts 0x%llx", pts);
292         if (m_use_streaminfo)
293         {
294                 if (pts >= m_pts_end && marg > 0 && m_end_valid)
295                         offset = m_offset_end;
296                 else
297                         offset = m_streaminfo.getAccessPoint(pts, marg);
298                 return 0;
299         } else
300         {
301                 calcBegin(); calcEnd();
302                 
303                 if (!m_begin_valid)
304                         return -1;
305                 if (!m_end_valid)
306                         return -1;
307
308                 if (!m_samples_taken)
309                         takeSamples();
310                 
311                 if (!m_samples.empty())
312                 {
313                         int maxtries = 5;
314                         pts_t p = -1;
315                         
316                         while (maxtries--)
317                         {
318                                         /* search entry before and after */
319                                 std::map<pts_t, off_t>::const_iterator l = m_samples.lower_bound(pts);
320                                 std::map<pts_t, off_t>::const_iterator u = l;
321
322                                 if (l != m_samples.begin())
323                                         --l;
324                                 
325                                         /* we could have seeked beyond the end */
326                                 if (u == m_samples.end())
327                                 {
328                                                 /* use last segment for interpolation. */
329                                         if (l != m_samples.begin())
330                                         {
331                                                 --u;
332                                                 --l;
333                                         }
334                                 }
335                                         
336                                         /* if we don't have enough points */
337                                 if (u == m_samples.end())
338                                         break;
339                                 
340                                 pts_t pts_diff = u->first - l->first;
341                                 off_t offset_diff = u->second - l->second;
342                                 
343                                 if (offset_diff < 0)
344                                 {
345                                         eDebug("something went wrong when taking samples.");
346                                         m_samples.clear();
347                                         takeSamples();
348                                         continue;
349                                 }
350
351                                 eDebug("using: %llx:%llx -> %llx:%llx", l->first, u->first, l->second, u->second);
352
353                                 int bitrate;
354                                 
355                                 if (pts_diff)
356                                         bitrate = offset_diff * 90000 * 8 / pts_diff;
357                                 else
358                                         bitrate = 0;
359
360                                 offset = l->second;
361                                 offset += ((pts - l->first) * (pts_t)bitrate) / 8ULL / 90000ULL;
362                                 offset -= offset % 188;
363                                 
364                                 p = pts;
365                                 
366                                 if (!takeSample(offset, p))
367                                 {
368                                         int diff = (p - pts) / 90;
369                         
370                                         eDebug("calculated diff %d ms", diff);
371                                         if (abs(diff) > 300)
372                                         {
373                                                 eDebug("diff to big, refining");
374                                                 continue;
375                                         }
376                                 } else
377                                         eDebug("no sample taken, refinement not possible.");
378
379                                 break;
380                         }
381                         
382                                 /* if even the first sample couldn't be taken, fall back. */
383                                 /* otherwise, return most refined result. */
384                         if (p != -1)
385                         {
386                                 pts = p;
387                                 eDebug("aborting. Taking %llx as offset for %lld", offset, pts);
388                                 return 0;
389                         }
390                 }
391                 
392                 int bitrate = calcBitrate();
393                 offset = pts * (pts_t)bitrate / 8ULL / 90000ULL;
394                 eDebug("fallback, bitrate=%d, results in %016llx", bitrate, offset);
395                 offset -= offset % 188;
396                 
397                 return 0;
398         }
399 }
400
401 int eDVBTSTools::getNextAccessPoint(pts_t &ts, const pts_t &start, int direction)
402 {
403         if (m_use_streaminfo)
404                 return m_streaminfo.getNextAccessPoint(ts, start, direction);
405         else
406         {
407                 eDebug("can't get next access point without streaminfo");
408                 return -1;
409         }
410 }
411
412 void eDVBTSTools::calcBegin()
413 {
414         if (!m_source || !m_source->valid())
415                 return;
416
417         if (!(m_begin_valid || m_futile))
418         {
419                 m_offset_begin = 0;
420                 if (!getPTS(m_offset_begin, m_pts_begin))
421                         m_begin_valid = 1;
422                 else
423                         m_futile = 1;
424         }
425 }
426
427 void eDVBTSTools::calcEnd()
428 {
429         if (!m_source || !m_source->valid())
430                 return;
431
432         off_t end = m_source->lseek(0, SEEK_END);
433         
434         if (llabs(end - m_last_filelength) > 1*1024*1024)
435         {
436                 m_last_filelength = end;
437                 m_end_valid = 0;
438                 
439                 m_futile = 0;
440 //              eDebug("file size changed, recalc length");
441         }
442         
443         int maxiter = 10;
444         
445         m_offset_end = m_last_filelength;
446         
447         while (!(m_end_valid || m_futile))
448         {
449                 if (!--maxiter)
450                 {
451                         m_futile = 1;
452                         return;
453                 }
454
455                 m_offset_end -= m_maxrange;
456                 if (m_offset_end < 0)
457                         m_offset_end = 0;
458
459                         /* restore offset if getpts fails */
460                 off_t off = m_offset_end;
461
462                 if (!getPTS(m_offset_end, m_pts_end))
463                         m_end_valid = 1;
464                 else
465                         m_offset_end = off;
466
467                 if (!m_offset_end)
468                 {
469                         m_futile = 1;
470                         break;
471                 }
472         }
473 }
474
475 int eDVBTSTools::calcLen(pts_t &len)
476 {
477         calcBegin(); calcEnd();
478         if (!(m_begin_valid && m_end_valid))
479                 return -1;
480         len = m_pts_end - m_pts_begin;
481                 /* wrap around? */
482         if (len < 0)
483                 len += 0x200000000LL;
484         return 0;
485 }
486
487 int eDVBTSTools::calcBitrate()
488 {
489         calcBegin(); calcEnd();
490         if (!(m_begin_valid && m_end_valid))
491                 return -1;
492
493         pts_t len_in_pts = m_pts_end - m_pts_begin;
494
495                 /* wrap around? */
496         if (len_in_pts < 0)
497                 len_in_pts += 0x200000000LL;
498         off_t len_in_bytes = m_offset_end - m_offset_begin;
499         
500         if (!len_in_pts)
501                 return -1;
502         
503         unsigned long long bitrate = len_in_bytes * 90000 * 8 / len_in_pts;
504         if ((bitrate < 10000) || (bitrate > 100000000))
505                 return -1;
506         
507         return bitrate;
508 }
509
510         /* pts, off */
511 void eDVBTSTools::takeSamples()
512 {
513         m_samples_taken = 1;
514         m_samples.clear();
515         pts_t dummy;
516         int retries=2;
517
518         if (calcLen(dummy) == -1)
519                 return;
520         
521         int nr_samples = 30;
522         off_t bytes_per_sample = (m_offset_end - m_offset_begin) / (long long)nr_samples;
523         if (bytes_per_sample < 40*1024*1024)
524                 bytes_per_sample = 40*1024*1024;
525
526         bytes_per_sample -= bytes_per_sample % 188;
527
528         eDebug("samples step %lld, pts begin %llx, pts end %llx, offs begin %lld, offs end %lld:",
529                 bytes_per_sample, m_pts_begin, m_pts_end, m_offset_begin, m_offset_end);
530
531         for (off_t offset = m_offset_begin; offset < m_offset_end;)
532         {
533                 pts_t p;
534                 if (takeSample(offset, p) && retries--)
535                         continue;
536                 retries = 2;
537                 offset += bytes_per_sample;
538         }
539         m_samples[0] = m_offset_begin;
540         m_samples[m_pts_end - m_pts_begin] = m_offset_end;
541 }
542
543         /* returns 0 when a sample was taken. */
544 int eDVBTSTools::takeSample(off_t off, pts_t &p)
545 {
546         off_t offset_org = off;
547
548         if (!eDVBTSTools::getPTS(off, p, 1))
549         {
550                         /* as we are happily mixing PTS and PCR values (no comment, please), we might
551                            end up with some "negative" segments. 
552                            
553                            so check if this new sample is between the previous and the next field*/
554
555                 std::map<pts_t, off_t>::const_iterator l = m_samples.lower_bound(p);
556                 std::map<pts_t, off_t>::const_iterator u = l;
557
558                 if (l != m_samples.begin())
559                 {
560                         --l;
561                         if (u != m_samples.end())
562                         {
563                                 if ((l->second > off) || (u->second < off))
564                                 {
565                                         eDebug("ignoring sample %lld %lld %lld (%llx %llx %llx)",
566                                                 l->second, off, u->second, l->first, p, u->first);
567                                         return 1;
568                                 }
569                         }
570                 }
571
572                 eDebug("adding sample %lld: pts 0x%llx -> pos %lld (diff %lld bytes)", offset_org, p, off, off-offset_org);
573                 m_samples[p] = off;
574                 return 0;
575         }
576         return -1;
577 }
578
579 int eDVBTSTools::findPMT(int &pmt_pid, int &service_id)
580 {
581                 /* FIXME: this will be factored out soon! */
582         if (!m_source || !m_source->valid())
583         {
584                 eDebug(" file not valid");
585                 return -1;
586         }
587
588         off_t position=0;
589
590         int left = 5*1024*1024;
591         
592         while (left >= 188)
593         {
594                 unsigned char packet[188];
595                 int ret = m_source->read(position, packet, 188);
596                 if (ret != 188)
597                 {
598                         eDebug("read error");
599                         break;
600                 }
601                 left -= 188;
602                 position += 188;
603
604                 if (packet[0] != 0x47)
605                 {
606                         int i = 0;
607                         while (i < 188)
608                         {
609                                 if (packet[i] == 0x47)
610                                         break;
611                                 --position;
612                                 ++i;
613                         }
614                         continue;
615                 }
616                 int pid = ((packet[1] << 8) | packet[2]) & 0x1FFF;
617                 
618                 int pusi = !!(packet[1] & 0x40);
619                 
620                 if (!pusi)
621                         continue;
622                 
623                         /* ok, now we have a PES header or section header*/
624                 unsigned char *sec;
625                 
626                         /* check for adaption field */
627                 if (packet[3] & 0x20)
628                 {
629                         if (packet[4] >= 183)
630                                 continue;
631                         sec = packet + packet[4] + 4 + 1;
632                 } else
633                         sec = packet + 4;
634                 
635                 if (sec[0])     /* table pointer, assumed to be 0 */
636                         continue;
637
638                 if (sec[1] == 0x02) /* program map section */
639                 {
640                         pmt_pid = pid;
641                         service_id = (sec[4] << 8) | sec[5];
642                         return 0;
643                 }
644         }
645         
646         return -1;
647 }
648
649 int eDVBTSTools::findFrame(off_t &_offset, size_t &len, int &direction, int frame_types)
650 {
651         off_t offset = _offset;
652         int nr_frames = 0;
653 //      eDebug("trying to find iFrame at %llx", offset);
654
655         if (!m_use_streaminfo)
656         {
657 //              eDebug("can't get next iframe without streaminfo");
658                 return -1;
659         }
660
661                                 /* let's find the iframe before the given offset */
662         unsigned long long data;
663         
664         if (direction < 0)
665                 offset--;
666
667         while (1)
668         {
669                 if (m_streaminfo.getStructureEntry(offset, data, (direction == 0) ? 1 : 0))
670                 {
671                         eDebug("getting structure info for origin offset failed.");
672                         return -1;
673                 }
674                 if (offset == 0x7fffffffffffffffLL) /* eof */
675                 {
676                         eDebug("reached eof");
677                         return -1;
678                 }
679                         /* data is usually the start code in the lower 8 bit, and the next byte <<8. we extract the picture type from there */
680                         /* we know that we aren't recording startcode 0x09 for mpeg2, so this is safe */
681                         /* TODO: check frame_types */
682                 int is_start = (data & 0xE0FF) == 0x0009; /* H.264 NAL unit access delimiter with I-frame*/
683                 is_start |= (data & 0x3800FF) == 0x080000; /* MPEG2 picture start code with I-frame */
684                 
685                 int is_frame = ((data & 0xFF) == 0x0009) || ((data & 0xFF) == 0x00); /* H.264 UAD or MPEG2 start code */
686                 
687                 if (is_frame)
688                 {
689                         if (direction < 0)
690                                 --nr_frames;
691                         else
692                                 ++nr_frames;
693                 }
694 //              eDebug("%08llx@%llx -> %d, %d", data, offset, is_start, nr_frames);
695                 if (is_start)
696                         break;
697
698                 if (direction == -1)
699                         --offset; /* move to previous entry */
700                 else if (direction == +1)
701                         direction = 0;
702         }
703                         /* let's find the next frame after the given offset */
704         off_t start = offset;
705
706         do {
707                 if (m_streaminfo.getStructureEntry(offset, data, 1))
708                 {
709                         eDebug("get next failed");
710                         return -1;
711                 }
712                 if (offset == 0x7fffffffffffffffLL) /* eof */
713                 {
714                         eDebug("reached eof (while looking for end of iframe)");
715                         return -1;
716                 }
717 //              eDebug("%08llx@%llx (next)", data, offset);
718         } while (((data & 0xFF) != 9) && ((data & 0xFF) != 0x00)); /* next frame */
719
720                         /* align to TS pkt start */
721 //      start = start - (start % 188);
722 //      offset = offset - (offset % 188);
723
724         len = offset - start;
725         _offset = start;
726         direction = nr_frames;
727 //      eDebug("result: offset=%llx, len: %ld", offset, (int)len);
728         return 0;
729 }
730
731 int eDVBTSTools::findNextPicture(off_t &offset, size_t &len, int &distance, int frame_types)
732 {
733         int nr_frames, direction;
734 //      eDebug("trying to move %d frames at %llx", distance, offset);
735         
736         frame_types = frametypeI; /* TODO: intelligent "allow IP frames when not crossing an I-Frame */
737
738         off_t new_offset = offset;
739         size_t new_len = len;
740         int first = 1;
741
742         if (distance > 0) {
743                 direction = 0;
744                 nr_frames = 0;
745         } else {
746                 direction = -1;
747                 nr_frames = -1;
748                 distance = -distance+1;
749         }       
750         while (distance > 0)
751         {
752                 int dir = direction;
753                 if (findFrame(new_offset, new_len, dir, frame_types))
754                 {
755 //                      eDebug("findFrame failed!\n");
756                         return -1;
757                 }
758                 
759                 distance -= abs(dir);
760                 
761 //              eDebug("we moved %d, %d to go frames (now at %llx)", dir, distance, new_offset);
762
763                 if (distance >= 0 || direction == 0)
764                 {
765                         first = 0;
766                         offset = new_offset;
767                         len = new_len;
768                         nr_frames += abs(dir);
769                 } 
770                 else if (first) {
771                         first = 0;
772                         offset = new_offset;
773                         len = new_len;
774                         nr_frames += abs(dir) + distance; // never jump forward during rewind
775                 }
776         }
777
778         distance = (direction < 0) ? -nr_frames : nr_frames;
779 //      eDebug("in total, we moved %d frames", nr_frames);
780
781         return 0;
782 }