Merge branch 'bug_570_playback_skip_fixes_and_cleanup_ml_aholst' into experimental
[vuplus_dvbapp] / lib / dvb / scan.cpp
1 #include <lib/dvb/idvb.h>
2 #include <dvbsi++/descriptor_tag.h>
3 #include <dvbsi++/service_descriptor.h>
4 #include <dvbsi++/satellite_delivery_system_descriptor.h>
5 #include <dvbsi++/terrestrial_delivery_system_descriptor.h>
6 #include <dvbsi++/cable_delivery_system_descriptor.h>
7 #include <dvbsi++/ca_identifier_descriptor.h>
8 #include <dvbsi++/registration_descriptor.h>
9 #include <lib/dvb/specs.h>
10 #include <lib/dvb/esection.h>
11 #include <lib/dvb/scan.h>
12 #include <lib/dvb/frontend.h>
13 #include <lib/base/eenv.h>
14 #include <lib/base/eerror.h>
15 #include <lib/base/estring.h>
16 #include <lib/python/python.h>
17 #include <errno.h>
18
19 #define SCAN_eDebug(x...) do { if (m_scan_debug) eDebug(x); } while(0)
20 #define SCAN_eDebugNoNewLine(x...) do { if (m_scan_debug) eDebugNoNewLine(x); } while(0)
21
22 DEFINE_REF(eDVBScan);
23
24 eDVBScan::eDVBScan(iDVBChannel *channel, bool usePAT, bool debug)
25         :m_channel(channel), m_channel_state(iDVBChannel::state_idle)
26         ,m_ready(0), m_ready_all(usePAT ? (readySDT|readyPAT) : readySDT)
27         ,m_pmt_running(false), m_abort_current_pmt(false), m_flags(0)
28         ,m_usePAT(usePAT), m_scan_debug(debug), m_show_add_tsid_onid_check_failed_msg(true)
29 {
30         if (m_channel->getDemux(m_demux))
31                 SCAN_eDebug("scan: failed to allocate demux!");
32         m_channel->connectStateChange(slot(*this, &eDVBScan::stateChange), m_stateChanged_connection);
33         std::string filename = eEnv::resolve("${sysconfdir}/scan_tp_valid_check.py");
34         FILE *f = fopen(filename.c_str(), "r");
35         if (f)
36         {
37                 char code[16384];
38                 size_t rd = fread(code, 1, 16383, f);
39                 if (rd)
40                 {
41                         code[rd]=0;
42                         m_additional_tsid_onid_check_func = Py_CompileString(code, filename.c_str(), Py_file_input);
43                 }
44                 fclose(f);
45         }
46 }
47
48 eDVBScan::~eDVBScan()
49 {
50         if (m_additional_tsid_onid_check_func)
51                 Py_DECREF(m_additional_tsid_onid_check_func);
52 }
53
54 int eDVBScan::isValidONIDTSID(int orbital_position, eOriginalNetworkID onid, eTransportStreamID tsid)
55 {
56         int ret;
57         switch (onid.get())
58         {
59         case 0:
60         case 0x1111:
61                 ret=0;
62                 break;
63         case 0x13E:  // workaround for 11258H and 11470V on hotbird with same ONID/TSID (0x13E/0x578)
64                 ret = orbital_position != 130 || tsid != 0x578;
65                 break;
66         case 1:
67                 ret = orbital_position == 192;
68                 break;
69         case 0x00B1:
70                 ret = tsid != 0x00B0;
71                 break;
72         case 0x00eb:
73                 ret = tsid != 0x4321;
74                 break;
75         case 0x0002:
76                 ret = abs(orbital_position-282) < 6 && tsid != 2019;
77                 // 12070H and 10936V have same tsid/onid.. but even the same services are provided
78                 break;
79         case 0x2000:
80                 ret = tsid != 0x1000;
81                 break;
82         case 0x5E: // Sirius 4.8E 12322V and 12226H
83                 ret = abs(orbital_position-48) < 3 && tsid != 1;
84                 break;
85         case 10100: // Eutelsat W7 36.0E 11644V and 11652V
86                 ret = orbital_position != 360 || tsid != 10187;
87                 break;
88         case 42: // Tuerksat 42.0E
89                 ret = orbital_position != 420 || (
90                     tsid != 8 && // 11830V 12729V
91                     tsid != 5 && // 12679V 12685H
92                     tsid != 2 && // 11096V 12015H
93                     tsid != 55); // 11996V 11716V
94                 break;
95         case 100: // Intelsat 10 68.5E 3808V 3796V 4012V, Amos 4.0W 10723V 11571H
96                 ret = (orbital_position != 685 && orbital_position != 3560) || tsid != 1;
97                 break;
98         case 70: // Thor 0.8W 11862H 12341V
99                 ret = abs(orbital_position-3592) < 3 && tsid != 46;
100                 break;
101         case 32: // NSS 806 (40.5W) 4059R, 3774L
102                 ret = orbital_position != 3195 || tsid != 21;
103                 break;
104         default:
105                 ret = onid.get() < 0xFF00;
106                 break;
107         }
108         if (ret && m_additional_tsid_onid_check_func)
109         {
110                 bool failed = true;
111                 ePyObject dict = PyDict_New();
112                 extern void PutToDict(ePyObject &, const char *, long);
113                 PyDict_SetItemString(dict, "__builtins__", PyEval_GetBuiltins());
114                 PutToDict(dict, "orbpos", orbital_position);
115                 PutToDict(dict, "tsid", tsid.get());
116                 PutToDict(dict, "onid", onid.get());
117                 ePyObject r = PyEval_EvalCode((PyCodeObject*)(PyObject*)m_additional_tsid_onid_check_func, dict, dict);
118                 if (r)
119                 {
120                         ePyObject o = PyDict_GetItemString(dict, "ret");
121                         if (o)
122                         {
123                                 if (PyInt_Check(o))
124                                 {
125                                         ret = PyInt_AsLong(o);
126                                         failed = false;
127                                 }
128                         }
129                         Py_DECREF(r);
130                 }
131                 if (failed && m_show_add_tsid_onid_check_failed_msg)
132                 {
133                         eDebug("execing /etc/enigma2/scan_tp_valid_check failed!\n"
134                                 "usable global variables in scan_tp_valid_check.py are 'orbpos', 'tsid', 'onid'\n"
135                                 "the return value must be stored in a global var named 'ret'");
136                         m_show_add_tsid_onid_check_failed_msg=false;
137                 }
138                 Py_DECREF(dict);
139         }
140         return ret;
141 }
142
143 eDVBNamespace eDVBScan::buildNamespace(eOriginalNetworkID onid, eTransportStreamID tsid, unsigned long hash)
144 {
145                 // on valid ONIDs, ignore frequency ("sub network") part
146         if (isValidONIDTSID((hash >> 16) & 0xFFFF, onid, tsid))
147                 hash &= ~0xFFFF;
148         return eDVBNamespace(hash);
149 }
150
151 void eDVBScan::stateChange(iDVBChannel *ch)
152 {
153         int state;
154         if (ch->getState(state))
155                 return;
156         if (m_channel_state == state)
157                 return;
158         
159         if (state == iDVBChannel::state_ok)
160         {
161                 startFilter();
162                 m_channel_state = state;
163         } else if (state == iDVBChannel::state_failed)
164         {
165                 m_ch_unavailable.push_back(m_ch_current);
166                 nextChannel();
167         }
168                         /* unavailable will timeout, anyway. */
169 }
170
171 RESULT eDVBScan::nextChannel()
172 {
173         ePtr<iDVBFrontend> fe;
174
175         m_SDT = 0; m_PAT = 0; m_BAT = 0; m_NIT = 0, m_PMT = 0;
176
177         m_ready = 0;
178
179         m_pat_tsid = eTransportStreamID();
180
181                 /* check what we need */
182         m_ready_all = readySDT;
183         
184         if (m_flags & scanNetworkSearch)
185                 m_ready_all |= readyNIT;
186         
187         if (m_flags & scanSearchBAT)
188                 m_ready_all |= readyBAT;
189
190         if (m_usePAT)
191                 m_ready_all |= readyPAT;
192
193         if (m_ch_toScan.empty())
194         {
195                 SCAN_eDebug("no channels left to scan.");
196                 SCAN_eDebug("%d channels scanned, %d were unavailable.", 
197                                 m_ch_scanned.size(), m_ch_unavailable.size());
198                 SCAN_eDebug("%d channels in database.", m_new_channels.size());
199                 m_event(evtFinish);
200                 return -ENOENT;
201         }
202         
203         m_ch_current = m_ch_toScan.front();
204         
205         m_ch_toScan.pop_front();
206         
207         if (m_channel->getFrontend(fe))
208         {
209                 m_event(evtFail);
210                 return -ENOTSUP;
211         }
212
213         m_chid_current = eDVBChannelID();
214
215         m_channel_state = iDVBChannel::state_idle;
216
217         if (fe->tune(*m_ch_current))
218                 return nextChannel();
219
220         m_event(evtUpdate);
221         return 0;
222 }
223
224 RESULT eDVBScan::startFilter()
225 {
226         bool startSDT=true;
227         ASSERT(m_demux);
228
229                         /* only start required filters filter */
230
231         if (m_ready_all & readyPAT)
232                 startSDT = m_ready & readyPAT;
233
234         // m_ch_current is not set, when eDVBScan is just used for a SDT update
235         if (!m_ch_current)
236         {
237                 unsigned int channelFlags;
238                 m_channel->getCurrentFrontendParameters(m_ch_current);
239                 m_ch_current->getFlags(channelFlags);
240                 if (channelFlags & iDVBFrontendParameters::flagOnlyFree)
241                         m_flags |= scanOnlyFree;
242         }
243
244         m_SDT = 0;
245         if (startSDT && (m_ready_all & readySDT))
246         {
247                 m_SDT = new eTable<ServiceDescriptionSection>(m_scan_debug);
248                 int tsid=-1;
249                 if (m_ready & readyPAT && m_ready & validPAT)
250                 {
251                         std::vector<ProgramAssociationSection*>::const_iterator i =
252                                 m_PAT->getSections().begin();
253                         ASSERT(i != m_PAT->getSections().end());
254                         tsid = (*i)->getTableIdExtension(); // in PAT this is the transport stream id
255                         m_pat_tsid = eTransportStreamID(tsid);
256                         for (; i != m_PAT->getSections().end(); ++i)
257                         {
258                                 const ProgramAssociationSection &pat = **i;
259                                 ProgramAssociationConstIterator program = pat.getPrograms()->begin();
260                                 for (; program != pat.getPrograms()->end(); ++program)
261                                         m_pmts_to_read.insert(std::pair<unsigned short, service>((*program)->getProgramNumber(), service((*program)->getProgramMapPid())));
262                         }
263                         m_PMT = new eTable<ProgramMapSection>(m_scan_debug);
264                         CONNECT(m_PMT->tableReady, eDVBScan::PMTready);
265                         PMTready(-2);
266                         // KabelBW HACK ... on 618Mhz and 626Mhz the transport stream id in PAT and SDT is different
267
268                         {
269                                 int type;
270                                 m_ch_current->getSystem(type);
271                                 if (type == iDVBFrontend::feCable)
272                                 {
273                                         eDVBFrontendParametersCable parm;
274                                         m_ch_current->getDVBC(parm);
275                                         if ((tsid == 0x00d7 && abs(parm.frequency-618000) < 2000) ||
276                                                 (tsid == 0x00d8 && abs(parm.frequency-626000) < 2000))
277                                                 tsid = -1;
278                                 }
279                         }
280                 }
281                 if (tsid == -1)
282                 {
283                         if (m_SDT->start(m_demux, eDVBSDTSpec()))
284                                 return -1;
285                 }
286                 else if (m_SDT->start(m_demux, eDVBSDTSpec(tsid, true)))
287                         return -1;
288                 CONNECT(m_SDT->tableReady, eDVBScan::SDTready);
289         }
290
291         if (!(m_ready & readyPAT))
292         {
293                 m_PAT = 0;
294                 if (m_ready_all & readyPAT)
295                 {
296                         m_PAT = new eTable<ProgramAssociationSection>(m_scan_debug);
297                         if (m_PAT->start(m_demux, eDVBPATSpec(4000)))
298                                 return -1;
299                         CONNECT(m_PAT->tableReady, eDVBScan::PATready);
300                 }
301
302                 m_NIT = 0;
303                 if (m_ready_all & readyNIT)
304                 {
305                         m_NIT = new eTable<NetworkInformationSection>(m_scan_debug);
306                         if (m_NIT->start(m_demux, eDVBNITSpec()))
307                                 return -1;
308                         CONNECT(m_NIT->tableReady, eDVBScan::NITready);
309                 }
310
311                 m_BAT = 0;
312                 if (m_ready_all & readyBAT)
313                 {
314                         m_BAT = new eTable<BouquetAssociationSection>(m_scan_debug);
315                         if (m_BAT->start(m_demux, eDVBBATSpec()))
316                                 return -1;
317                         CONNECT(m_BAT->tableReady, eDVBScan::BATready);
318                 }
319         }
320         return 0;
321 }
322
323 void eDVBScan::SDTready(int err)
324 {
325         SCAN_eDebug("got sdt %d", err);
326         m_ready |= readySDT;
327         if (!err)
328                 m_ready |= validSDT;
329         channelDone();
330 }
331
332 void eDVBScan::NITready(int err)
333 {
334         SCAN_eDebug("got nit, err %d", err);
335         m_ready |= readyNIT;
336         if (!err)
337                 m_ready |= validNIT;
338         channelDone();
339 }
340
341 void eDVBScan::BATready(int err)
342 {
343         SCAN_eDebug("got bat");
344         m_ready |= readyBAT;
345         if (!err)
346                 m_ready |= validBAT;
347         channelDone();
348 }
349
350 void eDVBScan::PATready(int err)
351 {
352         SCAN_eDebug("got pat");
353         m_ready |= readyPAT;
354         if (!err)
355                 m_ready |= validPAT;
356         startFilter(); // for starting the SDT filter
357 }
358
359 void eDVBScan::PMTready(int err)
360 {
361         SCAN_eDebug("got pmt %d", err);
362         if (!err)
363         {
364                 bool scrambled = false;
365                 bool have_audio = false;
366                 bool have_video = false;
367                 unsigned short pcrpid = 0xFFFF;
368                 std::vector<ProgramMapSection*>::const_iterator i;
369
370                 for (i = m_PMT->getSections().begin(); i != m_PMT->getSections().end(); ++i)
371                 {
372                         const ProgramMapSection &pmt = **i;
373                         if (pcrpid == 0xFFFF)
374                                 pcrpid = pmt.getPcrPid();
375                         else
376                                 SCAN_eDebug("already have a pcrpid %04x %04x", pcrpid, pmt.getPcrPid());
377                         ElementaryStreamInfoConstIterator es;
378                         for (es = pmt.getEsInfo()->begin(); es != pmt.getEsInfo()->end(); ++es)
379                         {
380                                 int isaudio = 0, isvideo = 0, is_scrambled = 0, forced_audio = 0, forced_video = 0;
381                                 switch ((*es)->getType())
382                                 {
383                                 case 0x1b: // AVC Video Stream (MPEG4 H264)
384                                 case 0x10: // MPEG 4 Part 2
385                                 case 0x01: // MPEG 1 video
386                                 case 0x02: // MPEG 2 video
387                                         isvideo = 1;
388                                         forced_video = 1;
389                                         //break; fall through !!!
390                                 case 0x03: // MPEG 1 audio
391                                 case 0x04: // MPEG 2 audio
392                                 case 0x0f: // MPEG 2 AAC
393                                 case 0x11: // MPEG 4 AAC
394                                         if (!isvideo) 
395                                         {
396                                                 forced_audio = 1;
397                                                 isaudio = 1;
398                                         }
399                                 case 0x06: // PES Private
400                                 case 0x81: // user private
401                                 case 0xEA: // TS_PSI_ST_SMPTE_VC1
402                                         for (DescriptorConstIterator desc = (*es)->getDescriptors()->begin();
403                                                         desc != (*es)->getDescriptors()->end(); ++desc)
404                                         {
405                                                 uint8_t tag = (*desc)->getTag();
406                                                 /* PES private can contain AC-3, DTS or lots of other stuff.
407                                                    check descriptors to get the exakt type. */
408                                                 if (!forced_video && !forced_audio)
409                                                 {
410                                                         switch (tag)
411                                                         {
412                                                         case 0x1C: // TS_PSI_DT_MPEG4_Audio
413                                                         case 0x2B: // TS_PSI_DT_MPEG2_AAC
414                                                         case AAC_DESCRIPTOR:
415                                                         case AC3_DESCRIPTOR:
416                                                         case DTS_DESCRIPTOR:
417                                                         case AUDIO_STREAM_DESCRIPTOR:
418                                                                 isaudio = 1;
419                                                                 break;
420                                                         case 0x28: // TS_PSI_DT_AVC
421                                                         case 0x1B: // TS_PSI_DT_MPEG4_Video
422                                                         case VIDEO_STREAM_DESCRIPTOR:
423                                                                 isvideo = 1;
424                                                                 break;
425                                                         case REGISTRATION_DESCRIPTOR: /* some services don't have a separate AC3 descriptor */
426                                                         {
427                                                                 RegistrationDescriptor *d = (RegistrationDescriptor*)(*desc);
428                                                                 switch (d->getFormatIdentifier())
429                                                                 {
430                                                                 case 0x44545331 ... 0x44545333: // DTS1/DTS2/DTS3
431                                                                 case 0x41432d33: // == 'AC-3'
432                                                                 case 0x42535344: // == 'BSSD' (LPCM)
433                                                                         isaudio = 1;
434                                                                         break;
435                                                                 case 0x56432d31: // == 'VC-1'
436                                                                         isvideo = 1;
437                                                                         break;
438                                                                 default:
439                                                                         break;
440                                                                 }
441                                                         }
442                                                         default:
443                                                                 break;
444                                                         }
445                                                 }
446                                                 if (tag == CA_DESCRIPTOR)
447                                                         is_scrambled = 1;
448                                         }
449                                 default:
450                                         break;
451                                 }
452                                 if (isvideo)
453                                         have_video = true;
454                                 else if (isaudio)
455                                         have_audio = true;
456                                 else
457                                         continue;
458                                 if (is_scrambled)
459                                         scrambled = true;
460                         }
461                         for (DescriptorConstIterator desc = pmt.getDescriptors()->begin();
462                                 desc != pmt.getDescriptors()->end(); ++desc)
463                         {
464                                 if ((*desc)->getTag() == CA_DESCRIPTOR)
465                                         scrambled = true;
466                         }
467                 }
468                 m_pmt_in_progress->second.scrambled = scrambled;
469                 if ( have_video )
470                         m_pmt_in_progress->second.serviceType = 1;
471                 else if ( have_audio )
472                         m_pmt_in_progress->second.serviceType = 2;
473                 else
474                         m_pmt_in_progress->second.serviceType = 100;
475         }
476         if (err == -1) // timeout or removed by sdt
477                 m_pmts_to_read.erase(m_pmt_in_progress++);
478         else if (m_pmt_running)
479                 ++m_pmt_in_progress;
480         else
481         {
482                 m_pmt_in_progress = m_pmts_to_read.begin();
483                 m_pmt_running = true;
484         }
485
486         if (m_pmt_in_progress != m_pmts_to_read.end())
487                 m_PMT->start(m_demux, eDVBPMTSpec(m_pmt_in_progress->second.pmtPid, m_pmt_in_progress->first, 4000));
488         else
489         {
490                 m_PMT = 0;
491                 m_pmt_running = false;
492                 channelDone();
493         }
494 }
495
496
497 void eDVBScan::addKnownGoodChannel(const eDVBChannelID &chid, iDVBFrontendParameters *feparm)
498 {
499                 /* add it to the list of known channels. */
500         if (chid)
501                 m_new_channels.insert(std::pair<eDVBChannelID,ePtr<iDVBFrontendParameters> >(chid, feparm));
502 }
503
504 void eDVBScan::addChannelToScan(const eDVBChannelID &chid, iDVBFrontendParameters *feparm)
505 {
506                 /* check if we don't already have that channel ... */
507
508         int type;
509         feparm->getSystem(type);
510
511         switch(type)
512         {
513         case iDVBFrontend::feSatellite:
514         {
515                 eDVBFrontendParametersSatellite parm;
516                 feparm->getDVBS(parm);
517                 SCAN_eDebug("try to add %d %d %d %d %d %d",
518                         parm.orbital_position, parm.frequency, parm.symbol_rate, parm.polarisation, parm.fec, parm.modulation);
519                 break;
520         }
521         case iDVBFrontend::feCable:
522         {
523                 eDVBFrontendParametersCable parm;
524                 feparm->getDVBC(parm);
525                 SCAN_eDebug("try to add %d %d %d %d",
526                         parm.frequency, parm.symbol_rate, parm.modulation, parm.fec_inner);
527                 break;
528         }
529         case iDVBFrontend::feTerrestrial:
530         {
531                 eDVBFrontendParametersTerrestrial parm;
532                 feparm->getDVBT(parm);
533                 SCAN_eDebug("try to add %d %d %d %d %d %d %d %d",
534                         parm.frequency, parm.modulation, parm.transmission_mode, parm.hierarchy,
535                         parm.guard_interval, parm.code_rate_LP, parm.code_rate_HP, parm.bandwidth);
536                 break;
537         }
538         }
539
540         int found_count=0;
541                 /* ... in the list of channels to scan */
542         for (std::list<ePtr<iDVBFrontendParameters> >::iterator i(m_ch_toScan.begin()); i != m_ch_toScan.end();)
543         {
544                 if (sameChannel(*i, feparm))
545                 {
546                         if (!found_count)
547                         {
548                                 *i = feparm;  // update
549                                 SCAN_eDebug("update");
550                         }
551                         else
552                         {
553                                 SCAN_eDebug("remove dupe");
554                                 m_ch_toScan.erase(i++);
555                                 continue;
556                         }
557                         ++found_count;
558                 }
559                 ++i;
560         }
561
562         if (found_count > 0)
563         {
564                 SCAN_eDebug("already in todo list");
565                 return;
566         }
567
568                 /* ... in the list of successfully scanned channels */
569         for (std::list<ePtr<iDVBFrontendParameters> >::const_iterator i(m_ch_scanned.begin()); i != m_ch_scanned.end(); ++i)
570                 if (sameChannel(*i, feparm))
571                 {
572                         SCAN_eDebug("successfully scanned");
573                         return;
574                 }
575
576                 /* ... in the list of unavailable channels */
577         for (std::list<ePtr<iDVBFrontendParameters> >::const_iterator i(m_ch_unavailable.begin()); i != m_ch_unavailable.end(); ++i)
578                 if (sameChannel(*i, feparm, true))
579                 {
580                         SCAN_eDebug("scanned but not available");
581                         return;
582                 }
583
584                 /* ... on the current channel */
585         if (sameChannel(m_ch_current, feparm))
586         {
587                 SCAN_eDebug("is current");
588                 return;
589         }
590
591         SCAN_eDebug("really add");
592                 /* otherwise, add it to the todo list. */
593         m_ch_toScan.push_front(feparm); // better.. then the rotor not turning wild from east to west :)
594 }
595
596 int eDVBScan::sameChannel(iDVBFrontendParameters *ch1, iDVBFrontendParameters *ch2, bool exact) const
597 {
598         int diff;
599         if (ch1->calculateDifference(ch2, diff, exact))
600                 return 0;
601         if (diff < 4000) // more than 4mhz difference?
602                 return 1;
603         return 0;
604 }
605
606 void eDVBScan::channelDone()
607 {
608         if (m_ready & validSDT && (!(m_flags & scanOnlyFree) || !m_pmt_running))
609         {
610                 unsigned long hash = 0;
611
612                 m_ch_current->getHash(hash);
613                 
614                 eDVBNamespace dvbnamespace = buildNamespace(
615                         (**m_SDT->getSections().begin()).getOriginalNetworkId(),
616                         (**m_SDT->getSections().begin()).getTransportStreamId(),
617                         hash);
618                 
619                 SCAN_eDebug("SDT: ");
620                 std::vector<ServiceDescriptionSection*>::const_iterator i;
621                 for (i = m_SDT->getSections().begin(); i != m_SDT->getSections().end(); ++i)
622                         processSDT(dvbnamespace, **i);
623                 m_ready &= ~validSDT;
624         }
625         
626         if (m_ready & validNIT)
627         {
628                 int system;
629                 std::list<ePtr<iDVBFrontendParameters> > m_ch_toScan_backup;
630                 m_ch_current->getSystem(system);
631                 SCAN_eDebug("dumping NIT");
632                 if (m_flags & clearToScanOnFirstNIT)
633                 {
634                         m_ch_toScan_backup = m_ch_toScan;
635                         m_ch_toScan.clear();
636                 }
637                 std::vector<NetworkInformationSection*>::const_iterator i;
638                 for (i = m_NIT->getSections().begin(); i != m_NIT->getSections().end(); ++i)
639                 {
640                         const TransportStreamInfoList &tsinfovec = *(*i)->getTsInfo();
641                         
642                         for (TransportStreamInfoConstIterator tsinfo(tsinfovec.begin()); 
643                                 tsinfo != tsinfovec.end(); ++tsinfo)
644                         {
645                                 SCAN_eDebug("TSID: %04x ONID: %04x", (*tsinfo)->getTransportStreamId(),
646                                         (*tsinfo)->getOriginalNetworkId());
647                                 
648                                 eOriginalNetworkID onid = (*tsinfo)->getOriginalNetworkId();
649                                 eTransportStreamID tsid = (*tsinfo)->getTransportStreamId();
650                                 
651                                 for (DescriptorConstIterator desc = (*tsinfo)->getDescriptors()->begin();
652                                                 desc != (*tsinfo)->getDescriptors()->end(); ++desc)
653                                 {
654                                         switch ((*desc)->getTag())
655                                         {
656                                         case CABLE_DELIVERY_SYSTEM_DESCRIPTOR:
657                                         {
658                                                 if (system != iDVBFrontend::feCable)
659                                                         break; // when current locked transponder is no cable transponder ignore this descriptor
660                                                 CableDeliverySystemDescriptor &d = (CableDeliverySystemDescriptor&)**desc;
661                                                 ePtr<eDVBFrontendParameters> feparm = new eDVBFrontendParameters;
662                                                 eDVBFrontendParametersCable cable;
663                                                 cable.set(d);
664                                                 feparm->setDVBC(cable);
665
666                                                 unsigned long hash=0;
667                                                 feparm->getHash(hash);
668                                                 eDVBNamespace ns = buildNamespace(onid, tsid, hash);
669
670                                                 addChannelToScan(
671                                                         eDVBChannelID(ns, tsid, onid),
672                                                         feparm);
673                                                 break;
674                                         }
675                                         case TERRESTRIAL_DELIVERY_SYSTEM_DESCRIPTOR:
676                                         {
677                                                 if (system != iDVBFrontend::feTerrestrial)
678                                                         break; // when current locked transponder is no terrestrial transponder ignore this descriptor
679                                                 TerrestrialDeliverySystemDescriptor &d = (TerrestrialDeliverySystemDescriptor&)**desc;
680                                                 ePtr<eDVBFrontendParameters> feparm = new eDVBFrontendParameters;
681                                                 eDVBFrontendParametersTerrestrial terr;
682                                                 terr.set(d);
683                                                 feparm->setDVBT(terr);
684
685                                                 unsigned long hash=0;
686                                                 feparm->getHash(hash);
687                                                 eDVBNamespace ns = buildNamespace(onid, tsid, hash);
688
689                                                 addChannelToScan(
690                                                         eDVBChannelID(ns, tsid, onid),
691                                                         feparm);
692                                                 break;
693                                         }
694                                         case SATELLITE_DELIVERY_SYSTEM_DESCRIPTOR:
695                                         {
696                                                 if (system != iDVBFrontend::feSatellite)
697                                                         break; // when current locked transponder is no satellite transponder ignore this descriptor
698
699                                                 SatelliteDeliverySystemDescriptor &d = (SatelliteDeliverySystemDescriptor&)**desc;
700                                                 if (d.getFrequency() < 10000)
701                                                         break;
702                                                 
703                                                 ePtr<eDVBFrontendParameters> feparm = new eDVBFrontendParameters;
704                                                 eDVBFrontendParametersSatellite sat;
705                                                 sat.set(d);
706
707                                                 eDVBFrontendParametersSatellite p;
708                                                 m_ch_current->getDVBS(p);
709
710                                                 if ( abs(p.orbital_position - sat.orbital_position) < 5 )
711                                                         sat.orbital_position = p.orbital_position;
712
713                                                 if ( abs(abs(3600 - p.orbital_position) - sat.orbital_position) < 5 )
714                                                 {
715                                                         SCAN_eDebug("found transponder with incorrect west/east flag ... correct this");
716                                                         sat.orbital_position = p.orbital_position;
717                                                 }
718
719                                                 feparm->setDVBS(sat);
720
721                                                 if ( p.orbital_position != sat.orbital_position)
722                                                         SCAN_eDebug("dropping this transponder, it's on another satellite.");
723                                                 else
724                                                 {
725                                                         unsigned long hash=0;
726                                                         feparm->getHash(hash);
727                                                         addChannelToScan(
728                                                                         eDVBChannelID(buildNamespace(onid, tsid, hash), tsid, onid),
729                                                                         feparm);
730                                                 }
731                                                 break;
732                                         }
733                                         default:
734                                                 SCAN_eDebug("descr<%x>", (*desc)->getTag());
735                                                 break;
736                                         }
737                                 }
738                         }
739                         
740                 }
741
742                         /* a pitfall is to have the clearToScanOnFirstNIT-flag set, and having channels which have
743                            no or invalid NIT. this code will not erase the toScan list unless at least one valid entry
744                            has been found.
745
746                            This is not a perfect solution, as the channel could contain a partial NIT. Life's bad.
747                         */
748                 if (m_flags & clearToScanOnFirstNIT)
749                 {
750                         if (m_ch_toScan.empty())
751                         {
752                                 eWarning("clearToScanOnFirstNIT was set, but NIT is invalid. Refusing to stop scan.");
753                                 m_ch_toScan = m_ch_toScan_backup;
754                         } else
755                                 m_flags &= ~clearToScanOnFirstNIT;
756                 }
757                 m_ready &= ~validNIT;
758         }
759
760         if (m_pmt_running || (m_ready & m_ready_all) != m_ready_all)
761         {
762                 if (m_abort_current_pmt)
763                 {
764                         m_abort_current_pmt = false;
765                         PMTready(-1);
766                 }
767                 return;
768         }
769
770         SCAN_eDebug("channel done!");
771         
772                 /* if we had services on this channel, we declare
773                    this channels as "known good". add it.
774                    
775                    (TODO: not yet implemented)
776                    a NIT entry could have possible overridden
777                    our frontend data with more exact data.
778                    
779                    (TODO: not yet implemented)
780                    the tuning process could have lead to more
781                    exact data than the user entered.
782                    
783                    The channel id was probably corrected
784                    by the data written in the SDT. this is
785                    important, as "initial transponder lists"
786                    usually don't have valid CHIDs (and that's
787                    good).
788                    
789                    These are the reasons for adding the transponder
790                    here, and not before.
791                 */
792
793         int type;
794         if (m_ch_current->getSystem(type))
795                 type = -1;
796
797         for (m_pmt_in_progress = m_pmts_to_read.begin(); m_pmt_in_progress != m_pmts_to_read.end();)
798         {
799                 eServiceReferenceDVB ref;
800                 ePtr<eDVBService> service = new eDVBService;
801
802                 if (!m_chid_current)
803                 {
804                         unsigned long hash = 0;
805
806                         m_ch_current->getHash(hash);
807
808                         m_chid_current = eDVBChannelID(
809                                 buildNamespace(eOriginalNetworkID(0), m_pat_tsid, hash),
810                                 m_pat_tsid, eOriginalNetworkID(0));
811                 }
812
813                 if (m_pmt_in_progress->second.serviceType == 1)
814                         SCAN_eDebug("SID %04x is VIDEO", m_pmt_in_progress->first);
815                 else if (m_pmt_in_progress->second.serviceType == 2)
816                         SCAN_eDebug("SID %04x is AUDIO", m_pmt_in_progress->first);
817                 else
818                         SCAN_eDebug("SID %04x is DATA", m_pmt_in_progress->first);
819
820                 ref.set(m_chid_current);
821                 ref.setServiceID(m_pmt_in_progress->first);
822                 ref.setServiceType(m_pmt_in_progress->second.serviceType);
823
824                 if (type != -1)
825                 {
826                         char sname[255];
827                         char pname[255];
828                         memset(pname, 0, sizeof(pname));
829                         memset(sname, 0, sizeof(sname));
830                         switch(type)
831                         {
832                                 case iDVBFrontend::feSatellite:
833                                 {
834                                         eDVBFrontendParametersSatellite parm;
835                                         m_ch_current->getDVBS(parm);
836                                         snprintf(sname, 255, "%d%c SID 0x%02x",
837                                                         parm.frequency/1000,
838                                                         parm.polarisation ? 'V' : 'H',
839                                                         m_pmt_in_progress->first);
840                                         snprintf(pname, 255, "%s %s %d%c %d.%d°%c",
841                                                 parm.system ? "DVB-S2" : "DVB-S",
842                                                 parm.modulation == 1 ? "QPSK" : "8PSK",
843                                                 parm.frequency/1000,
844                                                 parm.polarisation ? 'V' : 'H',
845                                                 parm.orbital_position/10,
846                                                 parm.orbital_position%10,
847                                                 parm.orbital_position > 0 ? 'E' : 'W');
848                                         break;
849                                 }
850                                 case iDVBFrontend::feTerrestrial:
851                                 {
852                                         eDVBFrontendParametersTerrestrial parm;
853                                         m_ch_current->getDVBT(parm);
854                                         snprintf(sname, 255, "%d SID 0x%02x",
855                                                 parm.frequency/1000,
856                                                 m_pmt_in_progress->first);
857                                         break;
858                                 }
859                                 case iDVBFrontend::feCable:
860                                 {
861                                         eDVBFrontendParametersCable parm;
862                                         m_ch_current->getDVBC(parm);
863                                         snprintf(sname, 255, "%d SID 0x%02x",
864                                                 parm.frequency/1000,
865                                                 m_pmt_in_progress->first);
866                                         break;
867                                 }
868                         }
869                         SCAN_eDebug("name '%s', provider_name '%s'", sname, pname);
870                         service->m_service_name = convertDVBUTF8(sname);
871                         service->genSortName();
872                         service->m_provider_name = convertDVBUTF8(pname);
873                 }
874
875                 if (!(m_flags & scanOnlyFree) || !m_pmt_in_progress->second.scrambled) {
876                         SCAN_eDebug("add not scrambled!");
877                         std::pair<std::map<eServiceReferenceDVB, ePtr<eDVBService> >::iterator, bool> i =
878                                 m_new_services.insert(std::pair<eServiceReferenceDVB, ePtr<eDVBService> >(ref, service));
879                         if (i.second)
880                         {
881                                 m_last_service = i.first;
882                                 m_event(evtNewService);
883                         }
884                 }
885                 else
886                         SCAN_eDebug("dont add... is scrambled!");
887                 m_pmts_to_read.erase(m_pmt_in_progress++);
888         }
889
890         if (!m_chid_current)
891                 eWarning("SCAN: the current channel's ID was not corrected - not adding channel.");
892         else
893         {
894                 addKnownGoodChannel(m_chid_current, m_ch_current);
895                 if (m_chid_current)
896                 {
897                         switch(type)
898                         {
899                                 case iDVBFrontend::feSatellite:
900                                 case iDVBFrontend::feTerrestrial:
901                                 case iDVBFrontend::feCable:
902                                 {
903                                         ePtr<iDVBFrontend> fe;
904                                         if (!m_channel->getFrontend(fe))
905                                         {
906                                                 ePyObject tp_dict = PyDict_New();
907                                                 fe->getTransponderData(tp_dict, false);
908 //                                              eDebug("add tuner data for tsid %04x, onid %04x, ns %08x",
909 //                                                      m_chid_current.transport_stream_id.get(), m_chid_current.original_network_id.get(),
910 //                                                      m_chid_current.dvbnamespace.get());
911                                                 m_tuner_data.insert(std::pair<eDVBChannelID, ePyObjectWrapper>(m_chid_current, tp_dict));
912                                                 Py_DECREF(tp_dict);
913                                         }
914                                 }
915                                 default:
916                                         break;
917                         }
918                 }
919         }
920
921         m_ch_scanned.push_back(m_ch_current);
922
923         for (std::list<ePtr<iDVBFrontendParameters> >::iterator i(m_ch_toScan.begin()); i != m_ch_toScan.end();)
924         {
925                 if (sameChannel(*i, m_ch_current))
926                 {
927                         SCAN_eDebug("remove dupe 2");
928                         m_ch_toScan.erase(i++);
929                         continue;
930                 }
931                 ++i;
932         }
933         
934         nextChannel();
935 }
936
937 void eDVBScan::start(const eSmartPtrList<iDVBFrontendParameters> &known_transponders, int flags)
938 {
939         m_flags = flags;
940         m_ch_toScan.clear();
941         m_ch_scanned.clear();
942         m_ch_unavailable.clear();
943         m_new_channels.clear();
944         m_tuner_data.clear();
945         m_new_services.clear();
946         m_last_service = m_new_services.end();
947
948         for (eSmartPtrList<iDVBFrontendParameters>::const_iterator i(known_transponders.begin()); i != known_transponders.end(); ++i)
949         {
950                 bool exist=false;
951                 for (std::list<ePtr<iDVBFrontendParameters> >::const_iterator ii(m_ch_toScan.begin()); ii != m_ch_toScan.end(); ++ii)
952                 {
953                         if (sameChannel(*i, *ii, true))
954                         {
955                                 exist=true;
956                                 break;
957                         }
958                 }
959                 if (!exist)
960                         m_ch_toScan.push_back(*i);
961         }
962
963         nextChannel();
964 }
965
966 void eDVBScan::insertInto(iDVBChannelList *db, bool dontRemoveOldFlags)
967 {
968         if (m_flags & scanRemoveServices)
969         {
970                 bool clearTerrestrial=false;
971                 bool clearCable=false;
972                 std::set<unsigned int> scanned_sat_positions;
973                 
974                 std::list<ePtr<iDVBFrontendParameters> >::iterator it(m_ch_scanned.begin());
975                 for (;it != m_ch_scanned.end(); ++it)
976                 {
977                         if (m_flags & scanDontRemoveUnscanned)
978                                 db->removeServices(&(*(*it)));
979                         else
980                         {
981                                 int system;
982                                 (*it)->getSystem(system);
983                                 switch(system)
984                                 {
985                                         case iDVBFrontend::feSatellite:
986                                         {
987                                                 eDVBFrontendParametersSatellite sat_parm;
988                                                 (*it)->getDVBS(sat_parm);
989                                                 scanned_sat_positions.insert(sat_parm.orbital_position);
990                                                 break;
991                                         }
992                                         case iDVBFrontend::feTerrestrial:
993                                         {
994                                                 clearTerrestrial=true;
995                                                 break;
996                                         }
997                                         case iDVBFrontend::feCable:
998                                         {
999                                                 clearCable=true;
1000                                                 break;
1001                                         }
1002                                 }
1003                         }
1004                 }
1005
1006                 for (it=m_ch_unavailable.begin();it != m_ch_unavailable.end(); ++it)
1007                 {
1008                         if (m_flags & scanDontRemoveUnscanned)
1009                                 db->removeServices(&(*(*it)));
1010                         else
1011                         {
1012                                 int system;
1013                                 (*it)->getSystem(system);
1014                                 switch(system)
1015                                 {
1016                                         case iDVBFrontend::feSatellite:
1017                                         {
1018                                                 eDVBFrontendParametersSatellite sat_parm;
1019                                                 (*it)->getDVBS(sat_parm);
1020                                                 scanned_sat_positions.insert(sat_parm.orbital_position);
1021                                                 break;
1022                                         }
1023                                         case iDVBFrontend::feTerrestrial:
1024                                         {
1025                                                 clearTerrestrial=true;
1026                                                 break;
1027                                         }
1028                                         case iDVBFrontend::feCable:
1029                                         {
1030                                                 clearCable=true;
1031                                                 break;
1032                                         }
1033                                 }
1034                         }
1035                 }
1036
1037                 if (clearTerrestrial)
1038                 {
1039                         eDVBChannelID chid;
1040                         chid.dvbnamespace=0xEEEE0000;
1041                         db->removeServices(chid);
1042                 }
1043                 if (clearCable)
1044                 {
1045                         eDVBChannelID chid;
1046                         chid.dvbnamespace=0xFFFF0000;
1047                         db->removeServices(chid);
1048                 }
1049                 for (std::set<unsigned int>::iterator x(scanned_sat_positions.begin()); x != scanned_sat_positions.end(); ++x)
1050                 {
1051                         eDVBChannelID chid;
1052                         if (m_flags & scanDontRemoveFeeds)
1053                                 chid.dvbnamespace = eDVBNamespace((*x)<<16);
1054 //                      eDebug("remove %d %08x", *x, chid.dvbnamespace.get());
1055                         db->removeServices(chid, *x);
1056                 }
1057         }
1058
1059         for (std::map<eDVBChannelID, ePtr<iDVBFrontendParameters> >::const_iterator 
1060                         ch(m_new_channels.begin()); ch != m_new_channels.end(); ++ch)
1061         {
1062                 int system;
1063                 ch->second->getSystem(system);
1064                 std::map<eDVBChannelID, ePyObjectWrapper>::iterator it = m_tuner_data.find(ch->first);
1065
1066                 switch(system)
1067                 {
1068                         case iDVBFrontend::feTerrestrial:
1069                         {
1070                                 eDVBFrontendParameters *p = (eDVBFrontendParameters*)&(*ch->second);
1071                                 eDVBFrontendParametersTerrestrial parm;
1072                                 int freq = PyInt_AsLong(PyDict_GetItemString(it->second, "frequency"));
1073                                 p->getDVBT(parm);
1074 //                              eDebug("corrected freq for tsid %04x, onid %04x, ns %08x is %d, old was %d",
1075 //                                      ch->first.transport_stream_id.get(), ch->first.original_network_id.get(),
1076 //                                      ch->first.dvbnamespace.get(), freq, parm.frequency);
1077                                 parm.frequency = freq;
1078                                 p->setDVBT(parm);
1079                                 break;
1080                         }
1081                         case iDVBFrontend::feSatellite: // no update of any transponder parameter yet
1082                         case iDVBFrontend::feCable:
1083                                 break;
1084                 }
1085
1086                 if (m_flags & scanOnlyFree)
1087                 {
1088                         eDVBFrontendParameters *ptr = (eDVBFrontendParameters*)&(*ch->second);
1089                         ptr->setFlags(iDVBFrontendParameters::flagOnlyFree);
1090                 }
1091
1092                 db->addChannelToList(ch->first, ch->second);
1093         }
1094
1095         for (std::map<eServiceReferenceDVB, ePtr<eDVBService> >::const_iterator
1096                 service(m_new_services.begin()); service != m_new_services.end(); ++service)
1097         {
1098                 ePtr<eDVBService> dvb_service;
1099                 if (!db->getService(service->first, dvb_service))
1100                 {
1101                         if (dvb_service->m_flags & eDVBService::dxNoSDT)
1102                                 continue;
1103                         if (!(dvb_service->m_flags & eDVBService::dxHoldName))
1104                         {
1105                                 dvb_service->m_service_name = service->second->m_service_name;
1106                                 dvb_service->m_service_name_sort = service->second->m_service_name_sort;
1107                         }
1108                         dvb_service->m_provider_name = service->second->m_provider_name;
1109                         if (service->second->m_ca.size())
1110                                 dvb_service->m_ca = service->second->m_ca;
1111                         if (!dontRemoveOldFlags) // do not remove new found flags when not wished
1112                                 dvb_service->m_flags &= ~eDVBService::dxNewFound;
1113                 }
1114                 else
1115                 {
1116                         db->addService(service->first, service->second);
1117                         if (!(m_flags & scanRemoveServices))
1118                                 service->second->m_flags |= eDVBService::dxNewFound;
1119                 }
1120         }
1121 }
1122
1123 RESULT eDVBScan::processSDT(eDVBNamespace dvbnamespace, const ServiceDescriptionSection &sdt)
1124 {
1125         const ServiceDescriptionList &services = *sdt.getDescriptions();
1126         SCAN_eDebug("ONID: %04x", sdt.getOriginalNetworkId());
1127         eDVBChannelID chid(dvbnamespace, sdt.getTransportStreamId(), sdt.getOriginalNetworkId());
1128         
1129         /* save correct CHID for this channel */
1130         m_chid_current = chid;
1131
1132         for (ServiceDescriptionConstIterator s(services.begin()); s != services.end(); ++s)
1133         {
1134                 unsigned short service_id = (*s)->getServiceId();
1135                 SCAN_eDebugNoNewLine("SID %04x: ", service_id);
1136                 bool add = true;
1137
1138                 if (m_flags & scanOnlyFree)
1139                 {
1140                         std::map<unsigned short, service>::iterator it =
1141                                 m_pmts_to_read.find(service_id);
1142                         if (it != m_pmts_to_read.end())
1143                         {
1144                                 if (it->second.scrambled)
1145                                 {
1146                                         SCAN_eDebug("is scrambled!");
1147                                         add = false;
1148                                 }
1149                                 else
1150                                         SCAN_eDebug("is free");
1151                         }
1152                         else {
1153                                 SCAN_eDebug("not found in PAT.. so we assume it is scrambled!!");
1154                                 add = false;
1155                         }
1156                 }
1157
1158                 if (add)
1159                 {
1160                         eServiceReferenceDVB ref;
1161                         ePtr<eDVBService> service = new eDVBService;
1162
1163                         ref.set(chid);
1164                         ref.setServiceID(service_id);
1165
1166                         for (DescriptorConstIterator desc = (*s)->getDescriptors()->begin();
1167                                         desc != (*s)->getDescriptors()->end(); ++desc)
1168                         {
1169                                 switch ((*desc)->getTag())
1170                                 {
1171                                 case SERVICE_DESCRIPTOR:
1172                                 {
1173                                         ServiceDescriptor &d = (ServiceDescriptor&)**desc;
1174                                         ref.setServiceType(d.getServiceType());
1175                                         service->m_service_name = convertDVBUTF8(d.getServiceName());
1176                                         service->genSortName();
1177
1178                                         service->m_provider_name = convertDVBUTF8(d.getServiceProviderName());
1179                                         SCAN_eDebug("name '%s', provider_name '%s'", service->m_service_name.c_str(), service->m_provider_name.c_str());
1180                                         break;
1181                                 }
1182                                 case CA_IDENTIFIER_DESCRIPTOR:
1183                                 {
1184                                         CaIdentifierDescriptor &d = (CaIdentifierDescriptor&)**desc;
1185                                         const CaSystemIdList &caids = *d.getCaSystemIds();
1186                                         SCAN_eDebugNoNewLine("CA ");
1187                                         for (CaSystemIdList::const_iterator i(caids.begin()); i != caids.end(); ++i)
1188                                         {
1189                                                 SCAN_eDebugNoNewLine("%04x ", *i);
1190                                                 service->m_ca.push_front(*i);
1191                                         }
1192                                         SCAN_eDebug("");
1193                                         break;
1194                                 }
1195                                 default:
1196                                         SCAN_eDebug("descr<%x>", (*desc)->getTag());
1197                                         break;
1198                                 }
1199                         }
1200
1201                         std::pair<std::map<eServiceReferenceDVB, ePtr<eDVBService> >::iterator, bool> i =
1202                                 m_new_services.insert(std::pair<eServiceReferenceDVB, ePtr<eDVBService> >(ref, service));
1203
1204                         if (i.second)
1205                         {
1206                                 m_last_service = i.first;
1207                                 m_event(evtNewService);
1208                         }
1209                 }
1210                 if (m_pmt_running && m_pmt_in_progress->first == service_id)
1211                         m_abort_current_pmt = true;
1212                 else
1213                         m_pmts_to_read.erase(service_id);
1214         }
1215
1216         return 0;
1217 }
1218
1219 RESULT eDVBScan::connectEvent(const Slot1<void,int> &event, ePtr<eConnection> &connection)
1220 {
1221         connection = new eConnection(this, m_event.connect(event));
1222         return 0;
1223 }
1224
1225 void eDVBScan::getStats(int &transponders_done, int &transponders_total, int &services)
1226 {
1227         transponders_done = m_ch_scanned.size() + m_ch_unavailable.size();
1228         transponders_total = m_ch_toScan.size() + transponders_done;
1229         services = m_new_services.size();
1230 }
1231
1232 void eDVBScan::getLastServiceName(std::string &last_service_name)
1233 {
1234         if (m_last_service == m_new_services.end())
1235                 last_service_name = "";
1236         else
1237                 last_service_name = m_last_service->second->m_service_name;
1238 }
1239
1240 RESULT eDVBScan::getFrontend(ePtr<iDVBFrontend> &fe)
1241 {
1242         if (m_channel)
1243                 return m_channel->getFrontend(fe);
1244         fe = 0;
1245         return -1;
1246 }
1247
1248 RESULT eDVBScan::getCurrentTransponder(ePtr<iDVBFrontendParameters> &tp)
1249 {
1250         if (m_ch_current)
1251         {
1252                 tp = m_ch_current;
1253                 return 0;
1254         }
1255         tp = 0;
1256         return -1;
1257 }