drivers/xen/events/events_base.c

   1 /*
   2  * Xen event channels
   3  *
   4  * Xen models interrupts with abstract event channels.  Because each
   5  * domain gets 1024 event channels, but NR_IRQ is not that large, we
   6  * must dynamically map irqs<->event channels.  The event channels
   7  * interface with the rest of the kernel by defining a xen interrupt
   8  * chip.  When an event is received, it is mapped to an irq and sent
   9  * through the normal interrupt processing path.
  10  *
  11  * There are four kinds of events which can be mapped to an event
  12  * channel:
  13  *
  14  * 1. Inter-domain notifications.  This includes all the virtual
  15  *    device events, since they're driven by front-ends in another domain
  16  *    (typically dom0).
  17  * 2. VIRQs, typically used for timers.  These are per-cpu events.
  18  * 3. IPIs.
  19  * 4. PIRQs - Hardware interrupts.
  20  *
  21  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
  22  */
  23
  24 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
  25
  26 #include <linux/linkage.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/irq.h>
  29 #include <linux/module.h>
  30 #include <linux/string.h>
  31 #include <linux/bootmem.h>
  32 #include <linux/slab.h>
  33 #include <linux/irqnr.h>
  34 #include <linux/pci.h>
  35 #include <linux/spinlock.h>
  36 #include <linux/cpu.h>
  37 #include <linux/atomic.h>
  38 #include <linux/ktime.h>
  39
  40 #ifdef CONFIG_X86
  41 #include <asm/desc.h>
  42 #include <asm/ptrace.h>
  43 #include <asm/irq.h>
  44 #include <asm/idle.h>
  45 #include <asm/io_apic.h>
  46 #include <asm/i8259.h>
  47 #include <asm/xen/pci.h>
  48 #endif
  49 #include <asm/sync_bitops.h>
  50 #include <asm/xen/hypercall.h>
  51 #include <asm/xen/hypervisor.h>
  52 #include <xen/page.h>
  53
  54 #include <xen/xen.h>
  55 #include <xen/hvm.h>
  56 #include <xen/xen-ops.h>
  57 #include <xen/events.h>
  58 #include <xen/interface/xen.h>
  59 #include <xen/interface/event_channel.h>
  60 #include <xen/interface/hvm/hvm_op.h>
  61 #include <xen/interface/hvm/params.h>
  62 #include <xen/interface/physdev.h>
  63 #include <xen/interface/sched.h>
  64 #include <xen/interface/vcpu.h>
  65 #include <asm/hw_irq.h>
  66
  67 #include "events_internal.h"
  68
  69 #undef MODULE_PARAM_PREFIX
  70 #define MODULE_PARAM_PREFIX "xen."
  71
  72 static uint __read_mostly event_loop_timeout = 2;
  73 module_param(event_loop_timeout, uint, 0644);
  74
  75 static uint __read_mostly event_eoi_delay = 10;
  76 module_param(event_eoi_delay, uint, 0644);
  77
  78 const struct evtchn_ops *evtchn_ops;
  79
  80 /*
  81  * This lock protects updates to the following mapping and reference-count
  82  * arrays. The lock does not need to be acquired to read the mapping tables.
  83  */
  84 static DEFINE_MUTEX(irq_mapping_update_lock);
  85
  86 /*
  87  * Lock protecting event handling loop against removing event channels.
  88  * Adding of event channels is no issue as the associated IRQ becomes active
  89  * only after everything is setup (before request_[threaded_]irq() the handler
  90  * can't be entered for an event, as the event channel will be unmasked only
  91  * then).
  92  */
  93 static DEFINE_RWLOCK(evtchn_rwlock);
  94
  95 /*
  96  * Lock hierarchy:
  97  *
  98  * irq_mapping_update_lock
  99  *   evtchn_rwlock
 100  *     IRQ-desc lock
 101  *       percpu eoi_list_lock
 102  *         irq_info->lock
 103  */
 104
 105 static LIST_HEAD(xen_irq_list_head);
 106
 107 /* IRQ <-> VIRQ mapping. */
 108 static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
 109
 110 /* IRQ <-> IPI mapping */
 111 static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
 112
 113 int **evtchn_to_irq;
 114 #ifdef CONFIG_X86
 115 static unsigned long *pirq_eoi_map;
 116 #endif
 117 static bool (*pirq_needs_eoi)(unsigned irq);
 118
 119 #define EVTCHN_ROW(e)  (e / (PAGE_SIZE/sizeof(**evtchn_to_irq)))
 120 #define EVTCHN_COL(e)  (e % (PAGE_SIZE/sizeof(**evtchn_to_irq)))
 121 #define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq))
 122
 123 /* Xen will never allocate port zero for any purpose. */
 124 #define VALID_EVTCHN(chn)       ((chn) != 0)
 125
 126 static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY];
 127
 128 static struct irq_chip xen_dynamic_chip;
 129 static struct irq_chip xen_lateeoi_chip;
 130 static struct irq_chip xen_percpu_chip;
 131 static struct irq_chip xen_pirq_chip;
 132 static void enable_dynirq(struct irq_data *data);
 133 static void disable_dynirq(struct irq_data *data);
 134
 135 static DEFINE_PER_CPU(unsigned int, irq_epoch);
 136
 137 static void clear_evtchn_to_irq_row(int *evtchn_row)
 138 {
 139         unsigned col;
 140
 141         for (col = 0; col < EVTCHN_PER_ROW; col++)
 142                 WRITE_ONCE(evtchn_row[col], -1);
 143 }
 144
 145 static void clear_evtchn_to_irq_all(void)
 146 {
 147         unsigned row;
 148
 149         for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) {
 150                 if (evtchn_to_irq[row] == NULL)
 151                         continue;
 152                 clear_evtchn_to_irq_row(evtchn_to_irq[row]);
 153         }
 154 }
 155
 156 static int set_evtchn_to_irq(unsigned evtchn, unsigned irq)
 157 {
 158         unsigned row;
 159         unsigned col;
 160         int *evtchn_row;
 161
 162         if (evtchn >= xen_evtchn_max_channels())
 163                 return -EINVAL;
 164
 165         row = EVTCHN_ROW(evtchn);
 166         col = EVTCHN_COL(evtchn);
 167
 168         if (evtchn_to_irq[row] == NULL) {
 169                 /* Unallocated irq entries return -1 anyway */
 170                 if (irq == -1)
 171                         return 0;
 172
 173                 evtchn_row = (int *) __get_free_pages(GFP_KERNEL, 0);
 174                 if (evtchn_row == NULL)
 175                         return -ENOMEM;
 176
 177                 clear_evtchn_to_irq_row(evtchn_row);
 178
 179                 /*
 180                  * We've prepared an empty row for the mapping. If a different
 181                  * thread was faster inserting it, we can drop ours.
 182                  */
 183                 if (cmpxchg(&evtchn_to_irq[row], NULL, evtchn_row) != NULL)
 184                         free_page((unsigned long) evtchn_row);
 185         }
 186
 187         WRITE_ONCE(evtchn_to_irq[row][col], irq);
 188         return 0;
 189 }
 190
 191 int get_evtchn_to_irq(unsigned evtchn)
 192 {
 193         if (evtchn >= xen_evtchn_max_channels())
 194                 return -1;
 195         if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL)
 196                 return -1;
 197         return READ_ONCE(evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]);
 198 }
 199
 200 /* Get info for IRQ */
 201 struct irq_info *info_for_irq(unsigned irq)
 202 {
 203         if (irq < nr_legacy_irqs())
 204                 return legacy_info_ptrs[irq];
 205         else
 206                 return irq_get_chip_data(irq);
 207 }
 208
 209 static void set_info_for_irq(unsigned int irq, struct irq_info *info)
 210 {
 211         if (irq < nr_legacy_irqs())
 212                 legacy_info_ptrs[irq] = info;
 213         else
 214                 irq_set_chip_data(irq, info);
 215 }
 216
 217 /* Constructors for packed IRQ information. */
 218 static int xen_irq_info_common_setup(struct irq_info *info,
 219                                      unsigned irq,
 220                                      enum xen_irq_type type,
 221                                      unsigned evtchn,
 222                                      unsigned short cpu)
 223 {
 224         int ret;
 225
 226         BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
 227
 228         info->type = type;
 229         info->irq = irq;
 230         info->evtchn = evtchn;
 231         info->cpu = cpu;
 232         info->mask_reason = EVT_MASK_REASON_EXPLICIT;
 233         raw_spin_lock_init(&info->lock);
 234
 235         ret = set_evtchn_to_irq(evtchn, irq);
 236         if (ret < 0)
 237                 return ret;
 238
 239         irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
 240
 241         return xen_evtchn_port_setup(info);
 242 }
 243
 244 static int xen_irq_info_evtchn_setup(unsigned irq,
 245                                      unsigned evtchn)
 246 {
 247         struct irq_info *info = info_for_irq(irq);
 248
 249         return xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0);
 250 }
 251
 252 static int xen_irq_info_ipi_setup(unsigned cpu,
 253                                   unsigned irq,
 254                                   unsigned evtchn,
 255                                   enum ipi_vector ipi)
 256 {
 257         struct irq_info *info = info_for_irq(irq);
 258
 259         info->u.ipi = ipi;
 260
 261         per_cpu(ipi_to_irq, cpu)[ipi] = irq;
 262
 263         return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0);
 264 }
 265
 266 static int xen_irq_info_virq_setup(unsigned cpu,
 267                                    unsigned irq,
 268                                    unsigned evtchn,
 269                                    unsigned virq)
 270 {
 271         struct irq_info *info = info_for_irq(irq);
 272
 273         info->u.virq = virq;
 274
 275         per_cpu(virq_to_irq, cpu)[virq] = irq;
 276
 277         return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0);
 278 }
 279
 280 static int xen_irq_info_pirq_setup(unsigned irq,
 281                                    unsigned evtchn,
 282                                    unsigned pirq,
 283                                    unsigned gsi,
 284                                    uint16_t domid,
 285                                    unsigned char flags)
 286 {
 287         struct irq_info *info = info_for_irq(irq);
 288
 289         info->u.pirq.pirq = pirq;
 290         info->u.pirq.gsi = gsi;
 291         info->u.pirq.domid = domid;
 292         info->u.pirq.flags = flags;
 293
 294         return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0);
 295 }
 296
 297 static void xen_irq_info_cleanup(struct irq_info *info)
 298 {
 299         set_evtchn_to_irq(info->evtchn, -1);
 300         xen_evtchn_port_remove(info->evtchn, info->cpu);
 301         info->evtchn = 0;
 302 }
 303
 304 /*
 305  * Accessors for packed IRQ information.
 306  */
 307 unsigned int evtchn_from_irq(unsigned irq)
 308 {
 309         const struct irq_info *info = NULL;
 310
 311         if (likely(irq < nr_irqs))
 312                 info = info_for_irq(irq);
 313         if (!info)
 314                 return 0;
 315
 316         return info->evtchn;
 317 }
 318
 319 unsigned irq_from_evtchn(unsigned int evtchn)
 320 {
 321         return get_evtchn_to_irq(evtchn);
 322 }
 323 EXPORT_SYMBOL_GPL(irq_from_evtchn);
 324
 325 int irq_from_virq(unsigned int cpu, unsigned int virq)
 326 {
 327         return per_cpu(virq_to_irq, cpu)[virq];
 328 }
 329
 330 static enum ipi_vector ipi_from_irq(unsigned irq)
 331 {
 332         struct irq_info *info = info_for_irq(irq);
 333
 334         BUG_ON(info == NULL);
 335         BUG_ON(info->type != IRQT_IPI);
 336
 337         return info->u.ipi;
 338 }
 339
 340 static unsigned virq_from_irq(unsigned irq)
 341 {
 342         struct irq_info *info = info_for_irq(irq);
 343
 344         BUG_ON(info == NULL);
 345         BUG_ON(info->type != IRQT_VIRQ);
 346
 347         return info->u.virq;
 348 }
 349
 350 static unsigned pirq_from_irq(unsigned irq)
 351 {
 352         struct irq_info *info = info_for_irq(irq);
 353
 354         BUG_ON(info == NULL);
 355         BUG_ON(info->type != IRQT_PIRQ);
 356
 357         return info->u.pirq.pirq;
 358 }
 359
 360 static enum xen_irq_type type_from_irq(unsigned irq)
 361 {
 362         return info_for_irq(irq)->type;
 363 }
 364
 365 unsigned cpu_from_irq(unsigned irq)
 366 {
 367         return info_for_irq(irq)->cpu;
 368 }
 369
 370 unsigned int cpu_from_evtchn(unsigned int evtchn)
 371 {
 372         int irq = get_evtchn_to_irq(evtchn);
 373         unsigned ret = 0;
 374
 375         if (irq != -1)
 376                 ret = cpu_from_irq(irq);
 377
 378         return ret;
 379 }
 380
 381 static void do_mask(struct irq_info *info, u8 reason)
 382 {
 383         unsigned long flags;
 384
 385         raw_spin_lock_irqsave(&info->lock, flags);
 386
 387         if (!info->mask_reason)
 388                 mask_evtchn(info->evtchn);
 389
 390         info->mask_reason |= reason;
 391
 392         raw_spin_unlock_irqrestore(&info->lock, flags);
 393 }
 394
 395 static void do_unmask(struct irq_info *info, u8 reason)
 396 {
 397         unsigned long flags;
 398
 399         raw_spin_lock_irqsave(&info->lock, flags);
 400
 401         info->mask_reason &= ~reason;
 402
 403         if (!info->mask_reason)
 404                 unmask_evtchn(info->evtchn);
 405
 406         raw_spin_unlock_irqrestore(&info->lock, flags);
 407 }
 408
 409 #ifdef CONFIG_X86
 410 static bool pirq_check_eoi_map(unsigned irq)
 411 {
 412         return test_bit(pirq_from_irq(irq), pirq_eoi_map);
 413 }
 414 #endif
 415
 416 static bool pirq_needs_eoi_flag(unsigned irq)
 417 {
 418         struct irq_info *info = info_for_irq(irq);
 419         BUG_ON(info->type != IRQT_PIRQ);
 420
 421         return info->u.pirq.flags & PIRQ_NEEDS_EOI;
 422 }
 423
 424 static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
 425 {
 426         int irq = get_evtchn_to_irq(chn);
 427         struct irq_info *info = info_for_irq(irq);
 428
 429         BUG_ON(irq == -1);
 430 #ifdef CONFIG_SMP
 431         cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu));
 432 #endif
 433         xen_evtchn_port_bind_to_cpu(info, cpu);
 434
 435         info->cpu = cpu;
 436 }
 437
 438 static void xen_evtchn_mask_all(void)
 439 {
 440         unsigned int evtchn;
 441
 442         for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++)
 443                 mask_evtchn(evtchn);
 444 }
 445
 446 /**
 447  * notify_remote_via_irq - send event to remote end of event channel via irq
 448  * @irq: irq of event channel to send event to
 449  *
 450  * Unlike notify_remote_via_evtchn(), this is safe to use across
 451  * save/restore. Notifications on a broken connection are silently
 452  * dropped.
 453  */
 454 void notify_remote_via_irq(int irq)
 455 {
 456         int evtchn = evtchn_from_irq(irq);
 457
 458         if (VALID_EVTCHN(evtchn))
 459                 notify_remote_via_evtchn(evtchn);
 460 }
 461 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
 462
 463 struct lateeoi_work {
 464         struct delayed_work delayed;
 465         spinlock_t eoi_list_lock;
 466         struct list_head eoi_list;
 467 };
 468
 469 static DEFINE_PER_CPU(struct lateeoi_work, lateeoi);
 470
 471 static void lateeoi_list_del(struct irq_info *info)
 472 {
 473         struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
 474         unsigned long flags;
 475
 476         spin_lock_irqsave(&eoi->eoi_list_lock, flags);
 477         list_del_init(&info->eoi_list);
 478         spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
 479 }
 480
 481 static void lateeoi_list_add(struct irq_info *info)
 482 {
 483         struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
 484         struct irq_info *elem;
 485         u64 now = get_jiffies_64();
 486         unsigned long delay;
 487         unsigned long flags;
 488
 489         if (now < info->eoi_time)
 490                 delay = info->eoi_time - now;
 491         else
 492                 delay = 1;
 493
 494         spin_lock_irqsave(&eoi->eoi_list_lock, flags);
 495
 496         if (list_empty(&eoi->eoi_list)) {
 497                 list_add(&info->eoi_list, &eoi->eoi_list);
 498                 mod_delayed_work_on(info->eoi_cpu, system_wq,
 499                                     &eoi->delayed, delay);
 500         } else {
 501                 list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) {
 502                         if (elem->eoi_time <= info->eoi_time)
 503                                 break;
 504                 }
 505                 list_add(&info->eoi_list, &elem->eoi_list);
 506         }
 507
 508         spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
 509 }
 510
 511 static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious)
 512 {
 513         evtchn_port_t evtchn;
 514         unsigned int cpu;
 515         unsigned int delay = 0;
 516
 517         evtchn = info->evtchn;
 518         if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list))
 519                 return;
 520
 521         if (spurious) {
 522                 if ((1 << info->spurious_cnt) < (HZ << 2))
 523                         info->spurious_cnt++;
 524                 if (info->spurious_cnt > 1) {
 525                         delay = 1 << (info->spurious_cnt - 2);
 526                         if (delay > HZ)
 527                                 delay = HZ;
 528                         if (!info->eoi_time)
 529                                 info->eoi_cpu = smp_processor_id();
 530                         info->eoi_time = get_jiffies_64() + delay;
 531                 }
 532         } else {
 533                 info->spurious_cnt = 0;
 534         }
 535
 536         cpu = info->eoi_cpu;
 537         if (info->eoi_time &&
 538             (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) {
 539                 lateeoi_list_add(info);
 540                 return;
 541         }
 542
 543         info->eoi_time = 0;
 544
 545         /* is_active hasn't been reset yet, do it now. */
 546         smp_store_release(&info->is_active, 0);
 547         do_unmask(info, EVT_MASK_REASON_EOI_PENDING);
 548 }
 549
 550 static void xen_irq_lateeoi_worker(struct work_struct *work)
 551 {
 552         struct lateeoi_work *eoi;
 553         struct irq_info *info;
 554         u64 now = get_jiffies_64();
 555         unsigned long flags;
 556
 557         eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed);
 558
 559         read_lock_irqsave(&evtchn_rwlock, flags);
 560
 561         while (true) {
 562                 spin_lock(&eoi->eoi_list_lock);
 563
 564                 info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info,
 565                                                 eoi_list);
 566
 567                 if (info == NULL || now < info->eoi_time) {
 568                         spin_unlock(&eoi->eoi_list_lock);
 569                         break;
 570                 }
 571
 572                 list_del_init(&info->eoi_list);
 573
 574                 spin_unlock(&eoi->eoi_list_lock);
 575
 576                 info->eoi_time = 0;
 577
 578                 xen_irq_lateeoi_locked(info, false);
 579         }
 580
 581         if (info)
 582                 mod_delayed_work_on(info->eoi_cpu, system_wq,
 583                                     &eoi->delayed, info->eoi_time - now);
 584
 585         read_unlock_irqrestore(&evtchn_rwlock, flags);
 586 }
 587
 588 static void xen_cpu_init_eoi(unsigned int cpu)
 589 {
 590         struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu);
 591
 592         INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker);
 593         spin_lock_init(&eoi->eoi_list_lock);
 594         INIT_LIST_HEAD(&eoi->eoi_list);
 595 }
 596
 597 void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags)
 598 {
 599         struct irq_info *info;
 600         unsigned long flags;
 601
 602         read_lock_irqsave(&evtchn_rwlock, flags);
 603
 604         info = info_for_irq(irq);
 605
 606         if (info)
 607                 xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS);
 608
 609         read_unlock_irqrestore(&evtchn_rwlock, flags);
 610 }
 611 EXPORT_SYMBOL_GPL(xen_irq_lateeoi);
 612
 613 static void xen_irq_init(unsigned irq)
 614 {
 615         struct irq_info *info;
 616
 617 #ifdef CONFIG_SMP
 618         /* By default all event channels notify CPU#0. */
 619         cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(0));
 620 #endif
 621
 622         info = kzalloc(sizeof(*info), GFP_KERNEL);
 623         if (info == NULL)
 624                 panic("Unable to allocate metadata for IRQ%d\n", irq);
 625
 626         info->type = IRQT_UNBOUND;
 627         info->refcnt = -1;
 628
 629         set_info_for_irq(irq, info);
 630
 631         INIT_LIST_HEAD(&info->eoi_list);
 632         list_add_tail(&info->list, &xen_irq_list_head);
 633 }
 634
 635 static int __must_check xen_allocate_irqs_dynamic(int nvec)
 636 {
 637         int i, irq = irq_alloc_descs(-1, 0, nvec, -1);
 638
 639         if (irq >= 0) {
 640                 for (i = 0; i < nvec; i++)
 641                         xen_irq_init(irq + i);
 642         }
 643
 644         return irq;
 645 }
 646
 647 static inline int __must_check xen_allocate_irq_dynamic(void)
 648 {
 649
 650         return xen_allocate_irqs_dynamic(1);
 651 }
 652
 653 static int __must_check xen_allocate_irq_gsi(unsigned gsi)
 654 {
 655         int irq;
 656
 657         /*
 658          * A PV guest has no concept of a GSI (since it has no ACPI
 659          * nor access to/knowledge of the physical APICs). Therefore
 660          * all IRQs are dynamically allocated from the entire IRQ
 661          * space.
 662          */
 663         if (xen_pv_domain() && !xen_initial_domain())
 664                 return xen_allocate_irq_dynamic();
 665
 666         /* Legacy IRQ descriptors are already allocated by the arch. */
 667         if (gsi < nr_legacy_irqs())
 668                 irq = gsi;
 669         else
 670                 irq = irq_alloc_desc_at(gsi, -1);
 671
 672         xen_irq_init(irq);
 673
 674         return irq;
 675 }
 676
 677 static void xen_free_irq(unsigned irq)
 678 {
 679         struct irq_info *info = info_for_irq(irq);
 680         unsigned long flags;
 681
 682         if (WARN_ON(!info))
 683                 return;
 684
 685         write_lock_irqsave(&evtchn_rwlock, flags);
 686
 687         if (!list_empty(&info->eoi_list))
 688                 lateeoi_list_del(info);
 689
 690         list_del(&info->list);
 691
 692         set_info_for_irq(irq, NULL);
 693
 694         WARN_ON(info->refcnt > 0);
 695
 696         write_unlock_irqrestore(&evtchn_rwlock, flags);
 697
 698         kfree(info);
 699
 700         /* Legacy IRQ descriptors are managed by the arch. */
 701         if (irq < nr_legacy_irqs())
 702                 return;
 703
 704         irq_free_desc(irq);
 705 }
 706
 707 static void xen_evtchn_close(unsigned int port)
 708 {
 709         struct evtchn_close close;
 710
 711         close.port = port;
 712         if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
 713                 BUG();
 714 }
 715
 716 static void event_handler_exit(struct irq_info *info)
 717 {
 718         smp_store_release(&info->is_active, 0);
 719         clear_evtchn(info->evtchn);
 720 }
 721
 722 static void pirq_query_unmask(int irq)
 723 {
 724         struct physdev_irq_status_query irq_status;
 725         struct irq_info *info = info_for_irq(irq);
 726
 727         BUG_ON(info->type != IRQT_PIRQ);
 728
 729         irq_status.irq = pirq_from_irq(irq);
 730         if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
 731                 irq_status.flags = 0;
 732
 733         info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
 734         if (irq_status.flags & XENIRQSTAT_needs_eoi)
 735                 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
 736 }
 737
 738 static void eoi_pirq(struct irq_data *data)
 739 {
 740         struct irq_info *info = info_for_irq(data->irq);
 741         int evtchn = info ? info->evtchn : 0;
 742         struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
 743         int rc = 0;
 744
 745         if (!VALID_EVTCHN(evtchn))
 746                 return;
 747
 748         if (unlikely(irqd_is_setaffinity_pending(data)) &&
 749             likely(!irqd_irq_disabled(data))) {
 750                 do_mask(info, EVT_MASK_REASON_TEMPORARY);
 751
 752                 event_handler_exit(info);
 753
 754                 irq_move_masked_irq(data);
 755
 756                 do_unmask(info, EVT_MASK_REASON_TEMPORARY);
 757         } else
 758                 event_handler_exit(info);
 759
 760         if (pirq_needs_eoi(data->irq)) {
 761                 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
 762                 WARN_ON(rc);
 763         }
 764 }
 765
 766 static void mask_ack_pirq(struct irq_data *data)
 767 {
 768         disable_dynirq(data);
 769         eoi_pirq(data);
 770 }
 771
 772 static unsigned int __startup_pirq(unsigned int irq)
 773 {
 774         struct evtchn_bind_pirq bind_pirq;
 775         struct irq_info *info = info_for_irq(irq);
 776         int evtchn = evtchn_from_irq(irq);
 777         int rc;
 778
 779         BUG_ON(info->type != IRQT_PIRQ);
 780
 781         if (VALID_EVTCHN(evtchn))
 782                 goto out;
 783
 784         bind_pirq.pirq = pirq_from_irq(irq);
 785         /* NB. We are happy to share unless we are probing. */
 786         bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
 787                                         BIND_PIRQ__WILL_SHARE : 0;
 788         rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
 789         if (rc != 0) {
 790                 pr_warn("Failed to obtain physical IRQ %d\n", irq);
 791                 return 0;
 792         }
 793         evtchn = bind_pirq.port;
 794
 795         pirq_query_unmask(irq);
 796
 797         rc = set_evtchn_to_irq(evtchn, irq);
 798         if (rc)
 799                 goto err;
 800
 801         info->evtchn = evtchn;
 802         bind_evtchn_to_cpu(evtchn, 0);
 803
 804         rc = xen_evtchn_port_setup(info);
 805         if (rc)
 806                 goto err;
 807
 808 out:
 809         do_unmask(info, EVT_MASK_REASON_EXPLICIT);
 810
 811         eoi_pirq(irq_get_irq_data(irq));
 812
 813         return 0;
 814
 815 err:
 816         pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc);
 817         xen_evtchn_close(evtchn);
 818         return 0;
 819 }
 820
 821 static unsigned int startup_pirq(struct irq_data *data)
 822 {
 823         return __startup_pirq(data->irq);
 824 }
 825
 826 static void shutdown_pirq(struct irq_data *data)
 827 {
 828         unsigned int irq = data->irq;
 829         struct irq_info *info = info_for_irq(irq);
 830         unsigned evtchn = evtchn_from_irq(irq);
 831
 832         BUG_ON(info->type != IRQT_PIRQ);
 833
 834         if (!VALID_EVTCHN(evtchn))
 835                 return;
 836
 837         do_mask(info, EVT_MASK_REASON_EXPLICIT);
 838         xen_evtchn_close(evtchn);
 839         xen_irq_info_cleanup(info);
 840 }
 841
 842 static void enable_pirq(struct irq_data *data)
 843 {
 844         startup_pirq(data);
 845 }
 846
 847 static void disable_pirq(struct irq_data *data)
 848 {
 849         disable_dynirq(data);
 850 }
 851
 852 int xen_irq_from_gsi(unsigned gsi)
 853 {
 854         struct irq_info *info;
 855
 856         list_for_each_entry(info, &xen_irq_list_head, list) {
 857                 if (info->type != IRQT_PIRQ)
 858                         continue;
 859
 860                 if (info->u.pirq.gsi == gsi)
 861                         return info->irq;
 862         }
 863
 864         return -1;
 865 }
 866 EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
 867
 868 static void __unbind_from_irq(unsigned int irq)
 869 {
 870         int evtchn = evtchn_from_irq(irq);
 871         struct irq_info *info = info_for_irq(irq);
 872
 873         if (info->refcnt > 0) {
 874                 info->refcnt--;
 875                 if (info->refcnt != 0)
 876                         return;
 877         }
 878
 879         if (VALID_EVTCHN(evtchn)) {
 880                 unsigned int cpu = cpu_from_irq(irq);
 881
 882                 xen_evtchn_close(evtchn);
 883
 884                 switch (type_from_irq(irq)) {
 885                 case IRQT_VIRQ:
 886                         per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1;
 887                         break;
 888                 case IRQT_IPI:
 889                         per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1;
 890                         break;
 891                 default:
 892                         break;
 893                 }
 894
 895                 xen_irq_info_cleanup(info);
 896         }
 897
 898         xen_free_irq(irq);
 899 }
 900
 901 /*
 902  * Do not make any assumptions regarding the relationship between the
 903  * IRQ number returned here and the Xen pirq argument.
 904  *
 905  * Note: We don't assign an event channel until the irq actually started
 906  * up.  Return an existing irq if we've already got one for the gsi.
 907  *
 908  * Shareable implies level triggered, not shareable implies edge
 909  * triggered here.
 910  */
 911 int xen_bind_pirq_gsi_to_irq(unsigned gsi,
 912                              unsigned pirq, int shareable, char *name)
 913 {
 914         int irq = -1;
 915         struct physdev_irq irq_op;
 916         int ret;
 917
 918         mutex_lock(&irq_mapping_update_lock);
 919
 920         irq = xen_irq_from_gsi(gsi);
 921         if (irq != -1) {
 922                 pr_info("%s: returning irq %d for gsi %u\n",
 923                         __func__, irq, gsi);
 924                 goto out;
 925         }
 926
 927         irq = xen_allocate_irq_gsi(gsi);
 928         if (irq < 0)
 929                 goto out;
 930
 931         irq_op.irq = irq;
 932         irq_op.vector = 0;
 933
 934         /* Only the privileged domain can do this. For non-priv, the pcifront
 935          * driver provides a PCI bus that does the call to do exactly
 936          * this in the priv domain. */
 937         if (xen_initial_domain() &&
 938             HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
 939                 xen_free_irq(irq);
 940                 irq = -ENOSPC;
 941                 goto out;
 942         }
 943
 944         ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF,
 945                                shareable ? PIRQ_SHAREABLE : 0);
 946         if (ret < 0) {
 947                 __unbind_from_irq(irq);
 948                 irq = ret;
 949                 goto out;
 950         }
 951
 952         pirq_query_unmask(irq);
 953         /* We try to use the handler with the appropriate semantic for the
 954          * type of interrupt: if the interrupt is an edge triggered
 955          * interrupt we use handle_edge_irq.
 956          *
 957          * On the other hand if the interrupt is level triggered we use
 958          * handle_fasteoi_irq like the native code does for this kind of
 959          * interrupts.
 960          *
 961          * Depending on the Xen version, pirq_needs_eoi might return true
 962          * not only for level triggered interrupts but for edge triggered
 963          * interrupts too. In any case Xen always honors the eoi mechanism,
 964          * not injecting any more pirqs of the same kind if the first one
 965          * hasn't received an eoi yet. Therefore using the fasteoi handler
 966          * is the right choice either way.
 967          */
 968         if (shareable)
 969                 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
 970                                 handle_fasteoi_irq, name);
 971         else
 972                 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
 973                                 handle_edge_irq, name);
 974
 975 out:
 976         mutex_unlock(&irq_mapping_update_lock);
 977
 978         return irq;
 979 }
 980
 981 #ifdef CONFIG_PCI_MSI
 982 int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
 983 {
 984         int rc;
 985         struct physdev_get_free_pirq op_get_free_pirq;
 986
 987         op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
 988         rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
 989
 990         WARN_ONCE(rc == -ENOSYS,
 991                   "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
 992
 993         return rc ? -1 : op_get_free_pirq.pirq;
 994 }
 995
 996 int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
 997                              int pirq, int nvec, const char *name, domid_t domid)
 998 {
 999         int i, irq, ret;
1000
1001         mutex_lock(&irq_mapping_update_lock);
1002
1003         irq = xen_allocate_irqs_dynamic(nvec);
1004         if (irq < 0)
1005                 goto out;
1006
1007         for (i = 0; i < nvec; i++) {
1008                 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name);
1009
1010                 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid,
1011                                               i == 0 ? 0 : PIRQ_MSI_GROUP);
1012                 if (ret < 0)
1013                         goto error_irq;
1014         }
1015
1016         ret = irq_set_msi_desc(irq, msidesc);
1017         if (ret < 0)
1018                 goto error_irq;
1019 out:
1020         mutex_unlock(&irq_mapping_update_lock);
1021         return irq;
1022 error_irq:
1023         while (nvec--)
1024                 __unbind_from_irq(irq + nvec);
1025         mutex_unlock(&irq_mapping_update_lock);
1026         return ret;
1027 }
1028 #endif
1029
1030 int xen_destroy_irq(int irq)
1031 {
1032         struct physdev_unmap_pirq unmap_irq;
1033         struct irq_info *info = info_for_irq(irq);
1034         int rc = -ENOENT;
1035
1036         mutex_lock(&irq_mapping_update_lock);
1037
1038         /*
1039          * If trying to remove a vector in a MSI group different
1040          * than the first one skip the PIRQ unmap unless this vector
1041          * is the first one in the group.
1042          */
1043         if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) {
1044                 unmap_irq.pirq = info->u.pirq.pirq;
1045                 unmap_irq.domid = info->u.pirq.domid;
1046                 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
1047                 /* If another domain quits without making the pci_disable_msix
1048                  * call, the Xen hypervisor takes care of freeing the PIRQs
1049                  * (free_domain_pirqs).
1050                  */
1051                 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
1052                         pr_info("domain %d does not have %d anymore\n",
1053                                 info->u.pirq.domid, info->u.pirq.pirq);
1054                 else if (rc) {
1055                         pr_warn("unmap irq failed %d\n", rc);
1056                         goto out;
1057                 }
1058         }
1059
1060         xen_free_irq(irq);
1061
1062 out:
1063         mutex_unlock(&irq_mapping_update_lock);
1064         return rc;
1065 }
1066
1067 int xen_irq_from_pirq(unsigned pirq)
1068 {
1069         int irq;
1070
1071         struct irq_info *info;
1072
1073         mutex_lock(&irq_mapping_update_lock);
1074
1075         list_for_each_entry(info, &xen_irq_list_head, list) {
1076                 if (info->type != IRQT_PIRQ)
1077                         continue;
1078                 irq = info->irq;
1079                 if (info->u.pirq.pirq == pirq)
1080                         goto out;
1081         }
1082         irq = -1;
1083 out:
1084         mutex_unlock(&irq_mapping_update_lock);
1085
1086         return irq;
1087 }
1088
1089
1090 int xen_pirq_from_irq(unsigned irq)
1091 {
1092         return pirq_from_irq(irq);
1093 }
1094 EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
1095
1096 static int bind_evtchn_to_irq_chip(evtchn_port_t evtchn, struct irq_chip *chip)
1097 {
1098         int irq;
1099         int ret;
1100
1101         if (evtchn >= xen_evtchn_max_channels())
1102                 return -ENOMEM;
1103
1104         mutex_lock(&irq_mapping_update_lock);
1105
1106         irq = get_evtchn_to_irq(evtchn);
1107
1108         if (irq == -1) {
1109                 irq = xen_allocate_irq_dynamic();
1110                 if (irq < 0)
1111                         goto out;
1112
1113                 irq_set_chip_and_handler_name(irq, chip,
1114                                               handle_edge_irq, "event");
1115
1116                 ret = xen_irq_info_evtchn_setup(irq, evtchn);
1117                 if (ret < 0) {
1118                         __unbind_from_irq(irq);
1119                         irq = ret;
1120                         goto out;
1121                 }
1122                 /* New interdomain events are bound to VCPU 0. */
1123                 bind_evtchn_to_cpu(evtchn, 0);
1124         } else {
1125                 struct irq_info *info = info_for_irq(irq);
1126                 WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
1127         }
1128
1129 out:
1130         mutex_unlock(&irq_mapping_update_lock);
1131
1132         return irq;
1133 }
1134
1135 int bind_evtchn_to_irq(evtchn_port_t evtchn)
1136 {
1137         return bind_evtchn_to_irq_chip(evtchn, &xen_dynamic_chip);
1138 }
1139 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
1140
1141 int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn)
1142 {
1143         return bind_evtchn_to_irq_chip(evtchn, &xen_lateeoi_chip);
1144 }
1145 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq_lateeoi);
1146
1147 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
1148 {
1149         struct evtchn_bind_ipi bind_ipi;
1150         int evtchn, irq;
1151         int ret;
1152
1153         mutex_lock(&irq_mapping_update_lock);
1154
1155         irq = per_cpu(ipi_to_irq, cpu)[ipi];
1156
1157         if (irq == -1) {
1158                 irq = xen_allocate_irq_dynamic();
1159                 if (irq < 0)
1160                         goto out;
1161
1162                 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
1163                                               handle_percpu_irq, "ipi");
1164
1165                 bind_ipi.vcpu = cpu;
1166                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1167                                                 &bind_ipi) != 0)
1168                         BUG();
1169                 evtchn = bind_ipi.port;
1170
1171                 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
1172                 if (ret < 0) {
1173                         __unbind_from_irq(irq);
1174                         irq = ret;
1175                         goto out;
1176                 }
1177                 bind_evtchn_to_cpu(evtchn, cpu);
1178         } else {
1179                 struct irq_info *info = info_for_irq(irq);
1180                 WARN_ON(info == NULL || info->type != IRQT_IPI);
1181         }
1182
1183  out:
1184         mutex_unlock(&irq_mapping_update_lock);
1185         return irq;
1186 }
1187
1188 static int bind_interdomain_evtchn_to_irq_chip(unsigned int remote_domain,
1189                                                evtchn_port_t remote_port,
1190                                                struct irq_chip *chip)
1191 {
1192         struct evtchn_bind_interdomain bind_interdomain;
1193         int err;
1194
1195         bind_interdomain.remote_dom  = remote_domain;
1196         bind_interdomain.remote_port = remote_port;
1197
1198         err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1199                                           &bind_interdomain);
1200
1201         return err ? : bind_evtchn_to_irq_chip(bind_interdomain.local_port,
1202                                                chip);
1203 }
1204
1205 int bind_interdomain_evtchn_to_irq(unsigned int remote_domain,
1206                                    evtchn_port_t remote_port)
1207 {
1208         return bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port,
1209                                                    &xen_dynamic_chip);
1210 }
1211 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq);
1212
1213 int bind_interdomain_evtchn_to_irq_lateeoi(unsigned int remote_domain,
1214                                            evtchn_port_t remote_port)
1215 {
1216         return bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port,
1217                                                    &xen_lateeoi_chip);
1218 }
1219 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi);
1220
1221 static int find_virq(unsigned int virq, unsigned int cpu)
1222 {
1223         struct evtchn_status status;
1224         int port, rc = -ENOENT;
1225
1226         memset(&status, 0, sizeof(status));
1227         for (port = 0; port < xen_evtchn_max_channels(); port++) {
1228                 status.dom = DOMID_SELF;
1229                 status.port = port;
1230                 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
1231                 if (rc < 0)
1232                         continue;
1233                 if (status.status != EVTCHNSTAT_virq)
1234                         continue;
1235                 if (status.u.virq == virq && status.vcpu == cpu) {
1236                         rc = port;
1237                         break;
1238                 }
1239         }
1240         return rc;
1241 }
1242
1243 /**
1244  * xen_evtchn_nr_channels - number of usable event channel ports
1245  *
1246  * This may be less than the maximum supported by the current
1247  * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum
1248  * supported.
1249  */
1250 unsigned xen_evtchn_nr_channels(void)
1251 {
1252         return evtchn_ops->nr_channels();
1253 }
1254 EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels);
1255
1256 int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu)
1257 {
1258         struct evtchn_bind_virq bind_virq;
1259         int evtchn, irq, ret;
1260
1261         mutex_lock(&irq_mapping_update_lock);
1262
1263         irq = per_cpu(virq_to_irq, cpu)[virq];
1264
1265         if (irq == -1) {
1266                 irq = xen_allocate_irq_dynamic();
1267                 if (irq < 0)
1268                         goto out;
1269
1270                 if (percpu)
1271                         irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
1272                                                       handle_percpu_irq, "virq");
1273                 else
1274                         irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
1275                                                       handle_edge_irq, "virq");
1276
1277                 bind_virq.virq = virq;
1278                 bind_virq.vcpu = cpu;
1279                 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1280                                                 &bind_virq);
1281                 if (ret == 0)
1282                         evtchn = bind_virq.port;
1283                 else {
1284                         if (ret == -EEXIST)
1285                                 ret = find_virq(virq, cpu);
1286                         BUG_ON(ret < 0);
1287                         evtchn = ret;
1288                 }
1289
1290                 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1291                 if (ret < 0) {
1292                         __unbind_from_irq(irq);
1293                         irq = ret;
1294                         goto out;
1295                 }
1296
1297                 bind_evtchn_to_cpu(evtchn, cpu);
1298         } else {
1299                 struct irq_info *info = info_for_irq(irq);
1300                 WARN_ON(info == NULL || info->type != IRQT_VIRQ);
1301         }
1302
1303 out:
1304         mutex_unlock(&irq_mapping_update_lock);
1305
1306         return irq;
1307 }
1308
1309 static void unbind_from_irq(unsigned int irq)
1310 {
1311         mutex_lock(&irq_mapping_update_lock);
1312         __unbind_from_irq(irq);
1313         mutex_unlock(&irq_mapping_update_lock);
1314 }
1315
1316 static int bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn,
1317                                           irq_handler_t handler,
1318                                           unsigned long irqflags,
1319                                           const char *devname, void *dev_id,
1320                                           struct irq_chip *chip)
1321 {
1322         int irq, retval;
1323
1324         irq = bind_evtchn_to_irq_chip(evtchn, chip);
1325         if (irq < 0)
1326                 return irq;
1327         retval = request_irq(irq, handler, irqflags, devname, dev_id);
1328         if (retval != 0) {
1329                 unbind_from_irq(irq);
1330                 return retval;
1331         }
1332
1333         return irq;
1334 }
1335
1336 int bind_evtchn_to_irqhandler(evtchn_port_t evtchn,
1337                               irq_handler_t handler,
1338                               unsigned long irqflags,
1339                               const char *devname, void *dev_id)
1340 {
1341         return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags,
1342                                               devname, dev_id,
1343                                               &xen_dynamic_chip);
1344 }
1345 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1346
1347 int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn,
1348                                       irq_handler_t handler,
1349                                       unsigned long irqflags,
1350                                       const char *devname, void *dev_id)
1351 {
1352         return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags,
1353                                               devname, dev_id,
1354                                               &xen_lateeoi_chip);
1355 }
1356 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler_lateeoi);
1357
1358 static int bind_interdomain_evtchn_to_irqhandler_chip(
1359                 unsigned int remote_domain, evtchn_port_t remote_port,
1360                 irq_handler_t handler, unsigned long irqflags,
1361                 const char *devname, void *dev_id, struct irq_chip *chip)
1362 {
1363         int irq, retval;
1364
1365         irq = bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port,
1366                                                   chip);
1367         if (irq < 0)
1368                 return irq;
1369
1370         retval = request_irq(irq, handler, irqflags, devname, dev_id);
1371         if (retval != 0) {
1372                 unbind_from_irq(irq);
1373                 return retval;
1374         }
1375
1376         return irq;
1377 }
1378
1379 int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
1380                                           evtchn_port_t remote_port,
1381                                           irq_handler_t handler,
1382                                           unsigned long irqflags,
1383                                           const char *devname,
1384                                           void *dev_id)
1385 {
1386         return bind_interdomain_evtchn_to_irqhandler_chip(remote_domain,
1387                                 remote_port, handler, irqflags, devname,
1388                                 dev_id, &xen_dynamic_chip);
1389 }
1390 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler);
1391
1392 int bind_interdomain_evtchn_to_irqhandler_lateeoi(unsigned int remote_domain,
1393                                                   evtchn_port_t remote_port,
1394                                                   irq_handler_t handler,
1395                                                   unsigned long irqflags,
1396                                                   const char *devname,
1397                                                   void *dev_id)
1398 {
1399         return bind_interdomain_evtchn_to_irqhandler_chip(remote_domain,
1400                                 remote_port, handler, irqflags, devname,
1401                                 dev_id, &xen_lateeoi_chip);
1402 }
1403 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler_lateeoi);
1404
1405 int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
1406                             irq_handler_t handler,
1407                             unsigned long irqflags, const char *devname, void *dev_id)
1408 {
1409         int irq, retval;
1410
1411         irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU);
1412         if (irq < 0)
1413                 return irq;
1414         retval = request_irq(irq, handler, irqflags, devname, dev_id);
1415         if (retval != 0) {
1416                 unbind_from_irq(irq);
1417                 return retval;
1418         }
1419
1420         return irq;
1421 }
1422 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1423
1424 int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1425                            unsigned int cpu,
1426                            irq_handler_t handler,
1427                            unsigned long irqflags,
1428                            const char *devname,
1429                            void *dev_id)
1430 {
1431         int irq, retval;
1432
1433         irq = bind_ipi_to_irq(ipi, cpu);
1434         if (irq < 0)
1435                 return irq;
1436
1437         irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
1438         retval = request_irq(irq, handler, irqflags, devname, dev_id);
1439         if (retval != 0) {
1440                 unbind_from_irq(irq);
1441                 return retval;
1442         }
1443
1444         return irq;
1445 }
1446
1447 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1448 {
1449         struct irq_info *info = info_for_irq(irq);
1450
1451         if (WARN_ON(!info))
1452                 return;
1453         free_irq(irq, dev_id);
1454         unbind_from_irq(irq);
1455 }
1456 EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1457
1458 /**
1459  * xen_set_irq_priority() - set an event channel priority.
1460  * @irq:irq bound to an event channel.
1461  * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN.
1462  */
1463 int xen_set_irq_priority(unsigned irq, unsigned priority)
1464 {
1465         struct evtchn_set_priority set_priority;
1466
1467         set_priority.port = evtchn_from_irq(irq);
1468         set_priority.priority = priority;
1469
1470         return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority,
1471                                            &set_priority);
1472 }
1473 EXPORT_SYMBOL_GPL(xen_set_irq_priority);
1474
1475 int evtchn_make_refcounted(unsigned int evtchn)
1476 {
1477         int irq = get_evtchn_to_irq(evtchn);
1478         struct irq_info *info;
1479
1480         if (irq == -1)
1481                 return -ENOENT;
1482
1483         info = info_for_irq(irq);
1484
1485         if (!info)
1486                 return -ENOENT;
1487
1488         WARN_ON(info->refcnt != -1);
1489
1490         info->refcnt = 1;
1491
1492         return 0;
1493 }
1494 EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1495
1496 int evtchn_get(unsigned int evtchn)
1497 {
1498         int irq;
1499         struct irq_info *info;
1500         int err = -ENOENT;
1501
1502         if (evtchn >= xen_evtchn_max_channels())
1503                 return -EINVAL;
1504
1505         mutex_lock(&irq_mapping_update_lock);
1506
1507         irq = get_evtchn_to_irq(evtchn);
1508         if (irq == -1)
1509                 goto done;
1510
1511         info = info_for_irq(irq);
1512
1513         if (!info)
1514                 goto done;
1515
1516         err = -EINVAL;
1517         if (info->refcnt <= 0 || info->refcnt == SHRT_MAX)
1518                 goto done;
1519
1520         info->refcnt++;
1521         err = 0;
1522  done:
1523         mutex_unlock(&irq_mapping_update_lock);
1524
1525         return err;
1526 }
1527 EXPORT_SYMBOL_GPL(evtchn_get);
1528
1529 void evtchn_put(unsigned int evtchn)
1530 {
1531         int irq = get_evtchn_to_irq(evtchn);
1532         if (WARN_ON(irq == -1))
1533                 return;
1534         unbind_from_irq(irq);
1535 }
1536 EXPORT_SYMBOL_GPL(evtchn_put);
1537
1538 void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1539 {
1540         int irq;
1541
1542 #ifdef CONFIG_X86
1543         if (unlikely(vector == XEN_NMI_VECTOR)) {
1544                 int rc =  HYPERVISOR_vcpu_op(VCPUOP_send_nmi, cpu, NULL);
1545                 if (rc < 0)
1546                         printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc);
1547                 return;
1548         }
1549 #endif
1550         irq = per_cpu(ipi_to_irq, cpu)[vector];
1551         BUG_ON(irq < 0);
1552         notify_remote_via_irq(irq);
1553 }
1554
1555 struct evtchn_loop_ctrl {
1556         ktime_t timeout;
1557         unsigned count;
1558         bool defer_eoi;
1559 };
1560
1561 void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl)
1562 {
1563         int irq;
1564         struct irq_info *info;
1565
1566         irq = get_evtchn_to_irq(port);
1567         if (irq == -1)
1568                 return;
1569
1570         /*
1571          * Check for timeout every 256 events.
1572          * We are setting the timeout value only after the first 256
1573          * events in order to not hurt the common case of few loop
1574          * iterations. The 256 is basically an arbitrary value.
1575          *
1576          * In case we are hitting the timeout we need to defer all further
1577          * EOIs in order to ensure to leave the event handling loop rather
1578          * sooner than later.
1579          */
1580         if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) {
1581                 ktime_t kt = ktime_get();
1582
1583                 if (!ctrl->timeout.tv64) {
1584                         kt = ktime_add_ms(kt,
1585                                           jiffies_to_msecs(event_loop_timeout));
1586                         ctrl->timeout = kt;
1587                 } else if (kt.tv64 > ctrl->timeout.tv64) {
1588                         ctrl->defer_eoi = true;
1589                 }
1590         }
1591
1592         info = info_for_irq(irq);
1593         if (xchg_acquire(&info->is_active, 1))
1594                 return;
1595
1596         if (ctrl->defer_eoi) {
1597                 info->eoi_cpu = smp_processor_id();
1598                 info->irq_epoch = __this_cpu_read(irq_epoch);
1599                 info->eoi_time = get_jiffies_64() + event_eoi_delay;
1600         }
1601
1602         generic_handle_irq(irq);
1603 }
1604
1605 static DEFINE_PER_CPU(unsigned, xed_nesting_count);
1606
1607 static void __xen_evtchn_do_upcall(void)
1608 {
1609         struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
1610         int cpu = get_cpu();
1611         unsigned count;
1612         struct evtchn_loop_ctrl ctrl = { 0 };
1613
1614         read_lock(&evtchn_rwlock);
1615
1616         do {
1617                 vcpu_info->evtchn_upcall_pending = 0;
1618
1619                 if (__this_cpu_inc_return(xed_nesting_count) - 1)
1620                         goto out;
1621
1622                 xen_evtchn_handle_events(cpu, &ctrl);
1623
1624                 BUG_ON(!irqs_disabled());
1625
1626                 count = __this_cpu_read(xed_nesting_count);
1627                 __this_cpu_write(xed_nesting_count, 0);
1628         } while (count != 1 || vcpu_info->evtchn_upcall_pending);
1629
1630 out:
1631         read_unlock(&evtchn_rwlock);
1632
1633         /*
1634          * Increment irq_epoch only now to defer EOIs only for
1635          * xen_irq_lateeoi() invocations occurring from inside the loop
1636          * above.
1637          */
1638         __this_cpu_inc(irq_epoch);
1639
1640         put_cpu();
1641 }
1642
1643 void xen_evtchn_do_upcall(struct pt_regs *regs)
1644 {
1645         struct pt_regs *old_regs = set_irq_regs(regs);
1646
1647         irq_enter();
1648 #ifdef CONFIG_X86
1649         exit_idle();
1650         inc_irq_stat(irq_hv_callback_count);
1651 #endif
1652
1653         __xen_evtchn_do_upcall();
1654
1655         irq_exit();
1656         set_irq_regs(old_regs);
1657 }
1658
1659 void xen_hvm_evtchn_do_upcall(void)
1660 {
1661         __xen_evtchn_do_upcall();
1662 }
1663 EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
1664
1665 /* Rebind a new event channel to an existing irq. */
1666 void rebind_evtchn_irq(int evtchn, int irq)
1667 {
1668         struct irq_info *info = info_for_irq(irq);
1669
1670         if (WARN_ON(!info))
1671                 return;
1672
1673         /* Make sure the irq is masked, since the new event channel
1674            will also be masked. */
1675         disable_irq(irq);
1676
1677         mutex_lock(&irq_mapping_update_lock);
1678
1679         /* After resume the irq<->evtchn mappings are all cleared out */
1680         BUG_ON(get_evtchn_to_irq(evtchn) != -1);
1681         /* Expect irq to have been bound before,
1682            so there should be a proper type */
1683         BUG_ON(info->type == IRQT_UNBOUND);
1684
1685         (void)xen_irq_info_evtchn_setup(irq, evtchn);
1686
1687         mutex_unlock(&irq_mapping_update_lock);
1688
1689         bind_evtchn_to_cpu(evtchn, info->cpu);
1690         /* This will be deferred until interrupt is processed */
1691         irq_set_affinity(irq, cpumask_of(info->cpu));
1692
1693         /* Unmask the event channel. */
1694         enable_irq(irq);
1695 }
1696
1697 /* Rebind an evtchn so that it gets delivered to a specific cpu */
1698 static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
1699 {
1700         struct evtchn_bind_vcpu bind_vcpu;
1701         struct irq_info *info = info_for_irq(irq);
1702         int evtchn = info ? info->evtchn : 0;
1703
1704         if (!VALID_EVTCHN(evtchn))
1705                 return -1;
1706
1707         if (!xen_support_evtchn_rebind())
1708                 return -1;
1709
1710         /* Send future instances of this interrupt to other vcpu. */
1711         bind_vcpu.port = evtchn;
1712         bind_vcpu.vcpu = tcpu;
1713
1714         /*
1715          * Mask the event while changing the VCPU binding to prevent
1716          * it being delivered on an unexpected VCPU.
1717          */
1718         do_mask(info, EVT_MASK_REASON_TEMPORARY);
1719
1720         /*
1721          * If this fails, it usually just indicates that we're dealing with a
1722          * virq or IPI channel, which don't actually need to be rebound. Ignore
1723          * it, but don't do the xenlinux-level rebind in that case.
1724          */
1725         if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1726                 bind_evtchn_to_cpu(evtchn, tcpu);
1727
1728         do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1729
1730         return 0;
1731 }
1732
1733 static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1734                             bool force)
1735 {
1736         unsigned tcpu = cpumask_first_and(dest, cpu_online_mask);
1737
1738         return rebind_irq_to_cpu(data->irq, tcpu);
1739 }
1740
1741 static void enable_dynirq(struct irq_data *data)
1742 {
1743         struct irq_info *info = info_for_irq(data->irq);
1744         evtchn_port_t evtchn = info ? info->evtchn : 0;
1745
1746         if (VALID_EVTCHN(evtchn))
1747                 do_unmask(info, EVT_MASK_REASON_EXPLICIT);
1748 }
1749
1750 static void disable_dynirq(struct irq_data *data)
1751 {
1752         struct irq_info *info = info_for_irq(data->irq);
1753         evtchn_port_t evtchn = info ? info->evtchn : 0;
1754
1755         if (VALID_EVTCHN(evtchn))
1756                 do_mask(info, EVT_MASK_REASON_EXPLICIT);
1757 }
1758
1759 static void ack_dynirq(struct irq_data *data)
1760 {
1761         struct irq_info *info = info_for_irq(data->irq);
1762         evtchn_port_t evtchn = info ? info->evtchn : 0;
1763
1764         if (!VALID_EVTCHN(evtchn))
1765                 return;
1766
1767         if (unlikely(irqd_is_setaffinity_pending(data)) &&
1768             likely(!irqd_irq_disabled(data))) {
1769                 do_mask(info, EVT_MASK_REASON_TEMPORARY);
1770
1771                 event_handler_exit(info);
1772
1773                 irq_move_masked_irq(data);
1774
1775                 do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1776         } else
1777                 event_handler_exit(info);
1778 }
1779
1780 static void mask_ack_dynirq(struct irq_data *data)
1781 {
1782         disable_dynirq(data);
1783         ack_dynirq(data);
1784 }
1785
1786 static void lateeoi_ack_dynirq(struct irq_data *data)
1787 {
1788         struct irq_info *info = info_for_irq(data->irq);
1789         evtchn_port_t evtchn = info ? info->evtchn : 0;
1790
1791         if (!VALID_EVTCHN(evtchn))
1792                 return;
1793
1794         do_mask(info, EVT_MASK_REASON_EOI_PENDING);
1795
1796         if (unlikely(irqd_is_setaffinity_pending(data)) &&
1797             likely(!irqd_irq_disabled(data))) {
1798                 do_mask(info, EVT_MASK_REASON_TEMPORARY);
1799
1800                 clear_evtchn(evtchn);
1801
1802                 irq_move_masked_irq(data);
1803
1804                 do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1805         } else
1806                 clear_evtchn(evtchn);
1807 }
1808
1809 static void lateeoi_mask_ack_dynirq(struct irq_data *data)
1810 {
1811         struct irq_info *info = info_for_irq(data->irq);
1812         evtchn_port_t evtchn = info ? info->evtchn : 0;
1813
1814         if (VALID_EVTCHN(evtchn)) {
1815                 do_mask(info, EVT_MASK_REASON_EXPLICIT);
1816                 ack_dynirq(data);
1817         }
1818 }
1819
1820 static int retrigger_dynirq(struct irq_data *data)
1821 {
1822         struct irq_info *info = info_for_irq(data->irq);
1823         evtchn_port_t evtchn = info ? info->evtchn : 0;
1824
1825         if (!VALID_EVTCHN(evtchn))
1826                 return 0;
1827
1828         do_mask(info, EVT_MASK_REASON_TEMPORARY);
1829         set_evtchn(evtchn);
1830         do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1831
1832         return 1;
1833 }
1834
1835 static void restore_pirqs(void)
1836 {
1837         int pirq, rc, irq, gsi;
1838         struct physdev_map_pirq map_irq;
1839         struct irq_info *info;
1840
1841         list_for_each_entry(info, &xen_irq_list_head, list) {
1842                 if (info->type != IRQT_PIRQ)
1843                         continue;
1844
1845                 pirq = info->u.pirq.pirq;
1846                 gsi = info->u.pirq.gsi;
1847                 irq = info->irq;
1848
1849                 /* save/restore of PT devices doesn't work, so at this point the
1850                  * only devices present are GSI based emulated devices */
1851                 if (!gsi)
1852                         continue;
1853
1854                 map_irq.domid = DOMID_SELF;
1855                 map_irq.type = MAP_PIRQ_TYPE_GSI;
1856                 map_irq.index = gsi;
1857                 map_irq.pirq = pirq;
1858
1859                 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1860                 if (rc) {
1861                         pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1862                                 gsi, irq, pirq, rc);
1863                         xen_free_irq(irq);
1864                         continue;
1865                 }
1866
1867                 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1868
1869                 __startup_pirq(irq);
1870         }
1871 }
1872
1873 static void restore_cpu_virqs(unsigned int cpu)
1874 {
1875         struct evtchn_bind_virq bind_virq;
1876         int virq, irq, evtchn;
1877
1878         for (virq = 0; virq < NR_VIRQS; virq++) {
1879                 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1880                         continue;
1881
1882                 BUG_ON(virq_from_irq(irq) != virq);
1883
1884                 /* Get a new binding from Xen. */
1885                 bind_virq.virq = virq;
1886                 bind_virq.vcpu = cpu;
1887                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1888                                                 &bind_virq) != 0)
1889                         BUG();
1890                 evtchn = bind_virq.port;
1891
1892                 /* Record the new mapping. */
1893                 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1894                 bind_evtchn_to_cpu(evtchn, cpu);
1895         }
1896 }
1897
1898 static void restore_cpu_ipis(unsigned int cpu)
1899 {
1900         struct evtchn_bind_ipi bind_ipi;
1901         int ipi, irq, evtchn;
1902
1903         for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1904                 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1905                         continue;
1906
1907                 BUG_ON(ipi_from_irq(irq) != ipi);
1908
1909                 /* Get a new binding from Xen. */
1910                 bind_ipi.vcpu = cpu;
1911                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1912                                                 &bind_ipi) != 0)
1913                         BUG();
1914                 evtchn = bind_ipi.port;
1915
1916                 /* Record the new mapping. */
1917                 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
1918                 bind_evtchn_to_cpu(evtchn, cpu);
1919         }
1920 }
1921
1922 /* Clear an irq's pending state, in preparation for polling on it */
1923 void xen_clear_irq_pending(int irq)
1924 {
1925         struct irq_info *info = info_for_irq(irq);
1926         evtchn_port_t evtchn = info ? info->evtchn : 0;
1927
1928         if (VALID_EVTCHN(evtchn))
1929                 event_handler_exit(info);
1930 }
1931 EXPORT_SYMBOL(xen_clear_irq_pending);
1932 void xen_set_irq_pending(int irq)
1933 {
1934         int evtchn = evtchn_from_irq(irq);
1935
1936         if (VALID_EVTCHN(evtchn))
1937                 set_evtchn(evtchn);
1938 }
1939
1940 bool xen_test_irq_pending(int irq)
1941 {
1942         int evtchn = evtchn_from_irq(irq);
1943         bool ret = false;
1944
1945         if (VALID_EVTCHN(evtchn))
1946                 ret = test_evtchn(evtchn);
1947
1948         return ret;
1949 }
1950
1951 /* Poll waiting for an irq to become pending with timeout.  In the usual case,
1952  * the irq will be disabled so it won't deliver an interrupt. */
1953 void xen_poll_irq_timeout(int irq, u64 timeout)
1954 {
1955         evtchn_port_t evtchn = evtchn_from_irq(irq);
1956
1957         if (VALID_EVTCHN(evtchn)) {
1958                 struct sched_poll poll;
1959
1960                 poll.nr_ports = 1;
1961                 poll.timeout = timeout;
1962                 set_xen_guest_handle(poll.ports, &evtchn);
1963
1964                 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1965                         BUG();
1966         }
1967 }
1968 EXPORT_SYMBOL(xen_poll_irq_timeout);
1969 /* Poll waiting for an irq to become pending.  In the usual case, the
1970  * irq will be disabled so it won't deliver an interrupt. */
1971 void xen_poll_irq(int irq)
1972 {
1973         xen_poll_irq_timeout(irq, 0 /* no timeout */);
1974 }
1975
1976 /* Check whether the IRQ line is shared with other guests. */
1977 int xen_test_irq_shared(int irq)
1978 {
1979         struct irq_info *info = info_for_irq(irq);
1980         struct physdev_irq_status_query irq_status;
1981
1982         if (WARN_ON(!info))
1983                 return -ENOENT;
1984
1985         irq_status.irq = info->u.pirq.pirq;
1986
1987         if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1988                 return 0;
1989         return !(irq_status.flags & XENIRQSTAT_shared);
1990 }
1991 EXPORT_SYMBOL_GPL(xen_test_irq_shared);
1992
1993 void xen_irq_resume(void)
1994 {
1995         unsigned int cpu;
1996         struct irq_info *info;
1997
1998         /* New event-channel space is not 'live' yet. */
1999         xen_evtchn_mask_all();
2000         xen_evtchn_resume();
2001
2002         /* No IRQ <-> event-channel mappings. */
2003         list_for_each_entry(info, &xen_irq_list_head, list)
2004                 info->evtchn = 0; /* zap event-channel binding */
2005
2006         clear_evtchn_to_irq_all();
2007
2008         for_each_possible_cpu(cpu) {
2009                 restore_cpu_virqs(cpu);
2010                 restore_cpu_ipis(cpu);
2011         }
2012
2013         restore_pirqs();
2014 }
2015
2016 static struct irq_chip xen_dynamic_chip __read_mostly = {
2017         .name                   = "xen-dyn",
2018
2019         .irq_disable            = disable_dynirq,
2020         .irq_mask               = disable_dynirq,
2021         .irq_unmask             = enable_dynirq,
2022
2023         .irq_ack                = ack_dynirq,
2024         .irq_mask_ack           = mask_ack_dynirq,
2025
2026         .irq_set_affinity       = set_affinity_irq,
2027         .irq_retrigger          = retrigger_dynirq,
2028 };
2029
2030 static struct irq_chip xen_lateeoi_chip __read_mostly = {
2031         /* The chip name needs to contain "xen-dyn" for irqbalance to work. */
2032         .name                   = "xen-dyn-lateeoi",
2033
2034         .irq_disable            = disable_dynirq,
2035         .irq_mask               = disable_dynirq,
2036         .irq_unmask             = enable_dynirq,
2037
2038         .irq_ack                = lateeoi_ack_dynirq,
2039         .irq_mask_ack           = lateeoi_mask_ack_dynirq,
2040
2041         .irq_set_affinity       = set_affinity_irq,
2042         .irq_retrigger          = retrigger_dynirq,
2043 };
2044
2045 static struct irq_chip xen_pirq_chip __read_mostly = {
2046         .name                   = "xen-pirq",
2047
2048         .irq_startup            = startup_pirq,
2049         .irq_shutdown           = shutdown_pirq,
2050         .irq_enable             = enable_pirq,
2051         .irq_disable            = disable_pirq,
2052
2053         .irq_mask               = disable_dynirq,
2054         .irq_unmask             = enable_dynirq,
2055
2056         .irq_ack                = eoi_pirq,
2057         .irq_eoi                = eoi_pirq,
2058         .irq_mask_ack           = mask_ack_pirq,
2059
2060         .irq_set_affinity       = set_affinity_irq,
2061
2062         .irq_retrigger          = retrigger_dynirq,
2063 };
2064
2065 static struct irq_chip xen_percpu_chip __read_mostly = {
2066         .name                   = "xen-percpu",
2067
2068         .irq_disable            = disable_dynirq,
2069         .irq_mask               = disable_dynirq,
2070         .irq_unmask             = enable_dynirq,
2071
2072         .irq_ack                = ack_dynirq,
2073 };
2074
2075 int xen_set_callback_via(uint64_t via)
2076 {
2077         struct xen_hvm_param a;
2078         a.domid = DOMID_SELF;
2079         a.index = HVM_PARAM_CALLBACK_IRQ;
2080         a.value = via;
2081         return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
2082 }
2083 EXPORT_SYMBOL_GPL(xen_set_callback_via);
2084
2085 #ifdef CONFIG_XEN_PVHVM
2086 /* Vector callbacks are better than PCI interrupts to receive event
2087  * channel notifications because we can receive vector callbacks on any
2088  * vcpu and we don't need PCI support or APIC interactions. */
2089 void xen_callback_vector(void)
2090 {
2091         int rc;
2092         uint64_t callback_via;
2093         if (xen_have_vector_callback) {
2094                 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
2095                 rc = xen_set_callback_via(callback_via);
2096                 if (rc) {
2097                         pr_err("Request for Xen HVM callback vector failed\n");
2098                         xen_have_vector_callback = 0;
2099                         return;
2100                 }
2101                 pr_info("Xen HVM callback vector for event delivery is enabled\n");
2102                 /* in the restore case the vector has already been allocated */
2103                 if (!test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors))
2104                         alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
2105                                         xen_hvm_callback_vector);
2106         }
2107 }
2108 #else
2109 void xen_callback_vector(void) {}
2110 #endif
2111
2112 static bool fifo_events = true;
2113 module_param(fifo_events, bool, 0);
2114
2115 static int xen_evtchn_cpu_prepare(unsigned int cpu)
2116 {
2117         int ret = 0;
2118
2119         xen_cpu_init_eoi(cpu);
2120
2121         if (evtchn_ops->percpu_init)
2122                 ret = evtchn_ops->percpu_init(cpu);
2123
2124         return ret;
2125 }
2126
2127 static int xen_evtchn_cpu_dead(unsigned int cpu)
2128 {
2129         int ret = 0;
2130
2131         if (evtchn_ops->percpu_deinit)
2132                 ret = evtchn_ops->percpu_deinit(cpu);
2133
2134         return ret;
2135 }
2136
2137 static int evtchn_cpu_notification(struct notifier_block *self,
2138                                    unsigned long action, void *hcpu)
2139 {
2140         int cpu = (long)hcpu;
2141         int ret = 0;
2142
2143         switch (action) {
2144         case CPU_UP_PREPARE:
2145                 ret = xen_evtchn_cpu_prepare(cpu);
2146                 break;
2147         case CPU_DEAD:
2148                 ret = xen_evtchn_cpu_dead(cpu);
2149                 break;
2150         default:
2151                 break;
2152         }
2153
2154         return ret < 0 ? NOTIFY_BAD : NOTIFY_OK;
2155 }
2156
2157 static struct notifier_block evtchn_cpu_notifier = {
2158         .notifier_call  = evtchn_cpu_notification,
2159 };
2160
2161 void __init xen_init_IRQ(void)
2162 {
2163         int ret = -EINVAL;
2164
2165         if (fifo_events)
2166                 ret = xen_evtchn_fifo_init();
2167         if (ret < 0)
2168                 xen_evtchn_2l_init();
2169
2170         xen_cpu_init_eoi(smp_processor_id());
2171
2172         register_cpu_notifier(&evtchn_cpu_notifier);
2173
2174         evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()),
2175                                 sizeof(*evtchn_to_irq), GFP_KERNEL);
2176         BUG_ON(!evtchn_to_irq);
2177
2178         /* No event channels are 'live' right now. */
2179         xen_evtchn_mask_all();
2180
2181         pirq_needs_eoi = pirq_needs_eoi_flag;
2182
2183 #ifdef CONFIG_X86
2184         if (xen_pv_domain()) {
2185                 irq_ctx_init(smp_processor_id());
2186                 if (xen_initial_domain())
2187                         pci_xen_initial_domain();
2188         }
2189         if (xen_feature(XENFEAT_hvm_callback_vector))
2190                 xen_callback_vector();
2191
2192         if (xen_hvm_domain()) {
2193                 native_init_IRQ();
2194                 /* pci_xen_hvm_init must be called after native_init_IRQ so that
2195                  * __acpi_register_gsi can point at the right function */
2196                 pci_xen_hvm_init();
2197         } else {
2198                 int rc;
2199                 struct physdev_pirq_eoi_gmfn eoi_gmfn;
2200
2201                 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
2202                 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map);
2203                 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
2204                 /* TODO: No PVH support for PIRQ EOI */
2205                 if (rc != 0) {
2206                         free_page((unsigned long) pirq_eoi_map);
2207                         pirq_eoi_map = NULL;
2208                 } else
2209                         pirq_needs_eoi = pirq_check_eoi_map;
2210         }
2211 #endif
2212 }