akaros/kern/drivers/dev/sdiahci.c
<<
>>
Prefs
   1/*
   2 * This file is part of the UCB release of Plan 9. It is subject to the license
   3 * terms in the LICENSE file found in the top-level directory of this
   4 * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
   5 * part of the UCB release of Plan 9, including this file, may be copied,
   6 * modified, propagated, or distributed except according to the terms contained
   7 * in the LICENSE file.
   8 */
   9
  10/*
  11 * ahci serial ata driver
  12 * copyright \xC2\xA9 2007-8 coraid, inc.
  13 */
  14
  15#include <assert.h>
  16#include <cpio.h>
  17#include <error.h>
  18#include <kmalloc.h>
  19#include <kref.h>
  20#include <net/ip.h>
  21#include <pmap.h>
  22#include <sd.h>
  23#include <slab.h>
  24#include <smp.h>
  25#include <stdio.h>
  26#include <string.h>
  27
  28#include <ahci.h>
  29
  30enum { Vatiamd = 0x1002,
  31       Vintel = 0x8086,
  32       Vmarvell = 0x1b4b,
  33};
  34
  35#define iprintd(...)                                                           \
  36        do {                                                                   \
  37                if (prid)                                                      \
  38                        printd(__VA_ARGS__);                                   \
  39        } while (0)
  40#define aprintd(...)                                                           \
  41        do {                                                                   \
  42                if (datapi)                                                    \
  43                        printd(__VA_ARGS__);                                   \
  44        } while (0)
  45#define Tname(c) tname[(c)->type]
  46#define Intel(x) ((x)->pci->ven_id == Vintel)
  47
  48enum { NCtlr = 16,
  49       NCtlrdrv = 32,
  50       NDrive = NCtlr * NCtlrdrv,
  51
  52       Read = 0,
  53       Write,
  54
  55       Nms = 256, /* ms. between drive checks */
  56       Mphywait = 2 * 1024 / Nms - 1,
  57       Midwait = 16 * 1024 / Nms - 1,
  58       Mcomrwait = 64 * 1024 / Nms - 1,
  59
  60       Obs = 0xa0, /* obsolete device bits */
  61
  62       /*
  63        * if we get more than this many interrupts per tick for a drive,
  64        * either the hardware is broken or we've got a bug in this driver.
  65        */
  66       Maxintrspertick = 2000, /* was 1000 */
  67};
  68
  69/* pci space configuration */
  70enum { Pmap = 0x90,
  71       Ppcs = 0x91,
  72       Prev = 0xa8,
  73};
  74
  75enum { Tesb,
  76       Tich,
  77       Tsb600,
  78       Tunk,
  79};
  80
  81static char *tname[] = {
  82    "63xxesb",
  83    "ich",
  84    "sb600",
  85    "unknown",
  86};
  87
  88enum { Dnull,
  89       Dmissing,
  90       Dnew,
  91       Dready,
  92       Derror,
  93       Dreset,
  94       Doffline,
  95       Dportreset,
  96       Dlast,
  97};
  98
  99static char *diskstates[Dlast] = {
 100    "null", "missing", "new", "ready", "error", "reset", "offline", "portreset",
 101};
 102
 103enum { DMautoneg,
 104       DMsatai,
 105       DMsataii,
 106       DMsata3,
 107};
 108
 109static char *modename[] = {
 110    /* used in control messages */
 111    "auto",
 112    "satai",
 113    "sataii",
 114    "sata3",
 115};
 116static char *descmode[] = {
 117    /*  only printed */
 118    "auto",
 119    "sata 1",
 120    "sata 2",
 121    "sata 3",
 122};
 123
 124static char *flagname[] = {
 125    "llba", "smart", "power", "nop", "atapi", "atapi16",
 126};
 127
 128struct drive {
 129        spinlock_t Lock;
 130
 131        struct ctlr *ctlr;
 132        struct sdunit *unit;
 133        char name[10];
 134        void *port;
 135        struct aportm portm;
 136        struct aportc portc; /* redundant ptr to port and portm */
 137
 138        unsigned char mediachange;
 139        unsigned char state;
 140        unsigned char smartrs;
 141
 142        uint64_t sectors;
 143        uint32_t secsize;
 144        uint32_t intick; /* start tick of current transfer */
 145        uint32_t lastseen;
 146        int wait;
 147        unsigned char mode; /* DMautoneg, satai or sataii */
 148        unsigned char active;
 149
 150        char serial[20 + 1];
 151        char firmware[8 + 1];
 152        char model[40 + 1];
 153
 154        int infosz;
 155        uint16_t *info;
 156        uint16_t tinyinfo[2]; /* used iff malloc fails */
 157
 158        int driveno; /* ctlr*NCtlrdrv + unit */
 159        /* controller port # != driveno when not all ports are enabled */
 160        int portno;
 161
 162        uint32_t lastintr0;
 163        uint32_t intrs;
 164};
 165
 166struct ctlr {
 167        spinlock_t Lock;
 168
 169        int type;
 170        int enabled;
 171        struct sdev *sdev;
 172        struct pci_device *pci;
 173        void *vector;
 174
 175        /* virtual register addresses */
 176        void *mmio;
 177        void *hba;
 178
 179        struct drive *rawdrive;
 180        struct drive *drive[NCtlrdrv];
 181        int ndrive;
 182        int mport; /* highest drive # (0-origin) on ich9 at least */
 183
 184        uint32_t lastintr0;
 185        uint32_t intrs; /* not attributable to any drive */
 186};
 187
 188struct Asleep {
 189        void *p;
 190        int i;
 191};
 192
 193struct sdifc sdiahciifc; // TODO(afergs): make static???
 194
 195static struct ctlr iactlr[NCtlr];
 196static struct sdev sdevs[NCtlr];
 197static int niactlr;
 198
 199static struct drive *iadrive[NDrive];
 200static int niadrive;
 201
 202/* these are fiddled in iawtopctl() */
 203static int debug;
 204static int prid = 1;
 205static int datapi;
 206
 207// TODO: does this get initialized correctly?
 208static char stab[] = {[0] = 'i',  'm', [8] = 't', 'c', 'p', 'e',
 209                      [16] = 'N', 'I', 'W',       'B', 'D', 'C',
 210                      'H',        'S', 'T',       'F', 'X'};
 211
 212// AHCI register access helper functions
 213// TODO(afergs): Figure out what the datatype for reg/offset should really be!
 214
 215static inline uint32_t ahci_hba_read32(void *base, uint32_t reg)
 216{
 217        return read_mmreg32((uintptr_t)base + reg);
 218}
 219
 220static inline void ahci_hba_write32(void *base, uint32_t reg, uint32_t val)
 221{
 222        write_mmreg32((uintptr_t)base + reg, val);
 223}
 224
 225static inline uint32_t ahci_port_read32(void *aport, uint32_t reg)
 226{
 227        return read_mmreg32((uintptr_t)aport + reg);
 228}
 229
 230static inline void ahci_port_write32(void *aport, uint32_t reg, uint32_t val)
 231{
 232        write_mmreg32((uintptr_t)aport + reg, val);
 233}
 234
 235static inline uint32_t ahci_list_read32(void *alist, uint32_t reg)
 236{
 237        return read_mmreg32((uintptr_t)alist + reg);
 238}
 239
 240static inline void ahci_list_write32(void *alist, uint32_t reg, uint32_t val)
 241{
 242        write_mmreg32((uintptr_t)alist + reg, val);
 243}
 244
 245static inline uint32_t ahci_prdt_read32(void *aprdt, uint32_t reg)
 246{
 247        return read_mmreg32((uintptr_t)aprdt + reg);
 248}
 249
 250static inline void ahci_prdt_write32(void *aprdt, uint32_t reg, uint32_t val)
 251{
 252        write_mmreg32((uintptr_t)aprdt + reg, val);
 253}
 254
 255static inline uint8_t ahci_cfis_read8(void *acfis, uint8_t offset)
 256{
 257        return read_mmreg8((uintptr_t)acfis + offset);
 258}
 259
 260static inline void ahci_cfis_write8(void *acfis, uint8_t offset, uint8_t val)
 261{
 262        write_mmreg8((uintptr_t)acfis + offset, val);
 263}
 264
 265static uint32_t set_bit32(void *base, uint32_t offset, uint32_t mask)
 266{
 267        uint32_t value = read_mmreg32((uintptr_t)base + offset);
 268
 269        value |= mask;
 270        write_mmreg32((uintptr_t)base + offset, value);
 271        return value;
 272}
 273
 274static uint32_t clear_bit32(void *base, uint32_t offset, uint32_t mask)
 275{
 276        uint32_t value = read_mmreg32((uintptr_t)base + offset);
 277
 278        value &= ~mask;
 279        write_mmreg32((uintptr_t)base + offset, value);
 280        return value;
 281}
 282
 283static uint8_t set_bit8(void *base, uint32_t offset, uint8_t mask)
 284{
 285        uint8_t value = read_mmreg8((uintptr_t)base + offset);
 286
 287        value |= mask;
 288        write_mmreg8((uintptr_t)base + offset, value);
 289        return value;
 290}
 291
 292static uint8_t clear_bit8(void *base, uint32_t offset, uint8_t mask)
 293{
 294        uint8_t value = read_mmreg8((uintptr_t)base + offset);
 295
 296        value &= ~mask;
 297        write_mmreg8((uintptr_t)base + offset, value);
 298        return value;
 299}
 300
 301/* ALL time units in this file are in milliseconds. */
 302static uint32_t ms(void)
 303{
 304        return (uint32_t)(epoch_nsec() / 1048576);
 305}
 306
 307/* TODO: if we like this, make it useable elsewhere. */
 308static void sdierror(struct cmdbuf *cb, char *fmt, ...)
 309{
 310        char *c = kzmalloc(512, MEM_WAIT);
 311        va_list ap;
 312
 313        assert(fmt);
 314        va_start(ap, fmt);
 315        vsnprintf(c, 512, fmt, ap);
 316        va_end(ap);
 317        cmderror(cb, c);
 318        kfree(c);
 319}
 320
 321static void serrstr(uint32_t r, char *s, char *e)
 322{
 323        int i;
 324
 325        e -= 3;
 326        for (i = 0; i < ARRAY_SIZE(stab) && s < e; i++)
 327                if (r & (1 << i) && stab[i]) {
 328                        *s++ = stab[i];
 329                        if (SerrBad & (1 << i))
 330                                *s++ = '*';
 331                }
 332        *s = 0;
 333}
 334
 335static char ntab[] = "0123456789abcdef";
 336
 337static void preg(volatile unsigned char *reg, int n)
 338{
 339        int i;
 340        char buf[25 * 3 + 1], *e;
 341
 342        e = buf;
 343        for (i = 0; i < n; i++) {
 344                *e++ = ntab[reg[i] >> 4];
 345                *e++ = ntab[reg[i] & 0xf];
 346                *e++ = ' ';
 347        }
 348        *e++ = '\n';
 349        *e = 0;
 350        printd(buf);
 351}
 352
 353static void dreg(char *s, void *p)
 354{
 355        printd("ahci: %stask=%#lx; cmd=%#lx; ci=%#lx; is=%#lx\n", s,
 356               ahci_port_read32(p, PORT_TFD), ahci_port_read32(p, PORT_CMD),
 357               ahci_port_read32(p, PORT_CI), ahci_port_read32(p, PORT_IS));
 358}
 359
 360static void esleep(int ms)
 361{
 362        ERRSTACK(1);
 363
 364        if (waserror()) {
 365                poperror();
 366                return;
 367        }
 368        kthread_usleep(ms * 1000);
 369        poperror();
 370}
 371
 372static int ahciclear(void *v)
 373{
 374        struct Asleep *s;
 375
 376        s = v;
 377        return (ahci_port_read32(s->p, PORT_CI) & s->i) == 0;
 378}
 379
 380static void aesleep(struct aportm *pm, struct Asleep *a, int ms)
 381{
 382        ERRSTACK(1);
 383
 384        if (waserror()) {
 385                poperror();
 386                return;
 387        }
 388        rendez_sleep_timeout(&pm->Rendez, ahciclear, a, ms * 1000);
 389        poperror();
 390}
 391
 392static int ahciwait(struct aportc *c, int ms)
 393{
 394        struct Asleep as;
 395        void *port;
 396        uint32_t cmd, tfd;
 397
 398        port = c->p;
 399        cmd = ahci_port_read32(port, PORT_CMD);
 400        printd("ahci: %s: CMD=0x%08x\n", __func__, cmd);
 401
 402        // TODO: Set the correct CI bit for the slot, not always slot 0!
 403        ahci_port_write32(port, PORT_CI, 1);
 404        as.p = port;
 405        as.i = 1;
 406        aesleep(c->pm, &as, ms);
 407        tfd = ahci_port_read32(port, PORT_TFD);
 408        if (((tfd & 0x81) == 0) && // Not busy and not error
 409            (ahci_port_read32(port, PORT_CI) == 0))
 410                return 0;
 411        dreg("ahci: ahciwait: timeout ", c->p);
 412        return -1;
 413}
 414
 415/* fill in cfis boilerplate */
 416static void *cfissetup(struct aportc *pc)
 417{
 418        void *cfis;
 419
 420        cfis = pc->pm->ctab; // cfis is the first thing in ctab, same location
 421        memset((void *)cfis, 0, 0x20);
 422        ahci_cfis_write8(cfis, 0, 0x27); // H2D
 423        ahci_cfis_write8(cfis, 1,
 424                         0x80); // Transfer due to update to CMD register
 425        ahci_cfis_write8(cfis, 7, Obs);
 426        return cfis;
 427}
 428
 429/* initialize pc's list */
 430static void listsetup(struct aportc *pc, int flags)
 431{
 432        void *list;
 433
 434        list = pc->pm->list;
 435        ahci_list_write32(list, ALIST_FLAGS, flags | 5);
 436        ahci_list_write32(list, ALIST_LEN, 0);
 437        ahci_list_write32(list, ALIST_CTAB, paddr_low32(pc->pm->ctab));
 438        ahci_list_write32(list, ALIST_CTABHI, paddr_high32(pc->pm->ctab));
 439        printd("ahci: %s: CTAB physical address=0x%08x:0x%08x\n", __func__,
 440               paddr_high32(pc->pm->ctab), paddr_low32(pc->pm->ctab));
 441}
 442
 443static int nop(struct aportc *pc)
 444{
 445        void *cfis;
 446
 447        if ((pc->pm->feat & Dnop) == 0)
 448                return -1;
 449        cfis = cfissetup(pc);
 450        ahci_cfis_write8(cfis, 2, 0);
 451        listsetup(pc, Lwrite);
 452        return ahciwait(pc, 3 * 1000);
 453}
 454
 455static int setfeatures(struct aportc *pc, unsigned char f)
 456{
 457        void *cfis;
 458
 459        cfis = cfissetup(pc);
 460        ahci_cfis_write8(cfis, 2, 0xef);
 461        ahci_cfis_write8(cfis, 3, f);
 462        listsetup(pc, Lwrite);
 463        return ahciwait(pc, 3 * 1000);
 464}
 465
 466static int setudmamode(struct aportc *pc, unsigned char f)
 467{
 468        void *cfis;
 469
 470        printd("ahci: %s: PORT_SIG=0x%08x\n", __func__,
 471               ahci_port_read32(pc->p, PORT_SIG));
 472        /* hack */
 473        if ((ahci_port_read32(pc->p, PORT_SIG) >> 16) == 0xeb14)
 474                return 0;
 475        cfis = cfissetup(pc);
 476        ahci_cfis_write8(cfis, 2, 0xef);
 477        ahci_cfis_write8(cfis, 3, 3);         /* set transfer mode */
 478        ahci_cfis_write8(cfis, 12, 0x40 | f); /* sector count */
 479        listsetup(pc, Lwrite);
 480        return ahciwait(pc, 3 * 1000);
 481}
 482
 483static void asleep(int ms)
 484{
 485        udelay(ms * 1000);
 486}
 487
 488static int ahciportreset(struct aportc *c)
 489{
 490        uint32_t cmd, i, scr_ctl;
 491        void *port;
 492
 493        port = c->p;
 494        clear_bit32(port, PORT_CMD, Afre | Ast);
 495        for (i = 0; i < 500; i += 25) {
 496                if ((ahci_port_read32(port, PORT_CMD) & Acr) == 0)
 497                        break;
 498                asleep(25);
 499        }
 500
 501        scr_ctl = ahci_port_read32(port, PORT_SCTL);
 502        scr_ctl = 1 | (scr_ctl & ~7);
 503        ahci_port_write32(port, PORT_SCTL, scr_ctl);
 504        printk("Sleeping one second\n");
 505        udelay(1000 * 1000);
 506        clear_bit32(port, PORT_SCTL, 7); // TODO: Make sure PxCMD.ST == 0 first?
 507        return 0;
 508}
 509
 510static int smart(struct aportc *pc, int n)
 511{
 512        void *cfis;
 513
 514        if ((pc->pm->feat & Dsmart) == 0)
 515                return -1;
 516        cfis = cfissetup(pc);
 517        ahci_cfis_write8(cfis, 2, 0xb0);
 518        ahci_cfis_write8(cfis, 3, 0xd8 + n); /* able smart */
 519        ahci_cfis_write8(cfis, 5, 0x4f);
 520        ahci_cfis_write8(cfis, 6, 0xc2);
 521        listsetup(pc, Lwrite);
 522        // TODO(afergs): Replace 1|32 with constants
 523        if (ahciwait(pc, 1000) == -1 ||
 524            ahci_port_read32(pc->p, PORT_TFD) & (1 | 32)) {
 525                printd("ahci: smart fail %#lx\n",
 526                       ahci_port_read32(pc->p, PORT_TFD));
 527                return -1;
 528        }
 529        if (n)
 530                return 0;
 531        return 1;
 532}
 533
 534static int smartrs(struct aportc *pc)
 535{
 536        void *cfis;
 537        volatile unsigned char *rfis;
 538        uint32_t tfd;
 539
 540        cfis = cfissetup(pc);
 541        ahci_cfis_write8(cfis, 2, 0xb0);
 542        ahci_cfis_write8(cfis, 3, 0xda); /* return smart status */
 543        ahci_cfis_write8(cfis, 5, 0x4f);
 544        ahci_cfis_write8(cfis, 6, 0xc2);
 545        listsetup(pc, Lwrite);
 546
 547        rfis = pc->pm->fis.r;
 548        tfd = ahci_port_read32(pc->p, PORT_TFD);
 549        // TODO(afergs): Replace 1|32 with constants
 550        if (ahciwait(pc, 1000) == -1 || tfd & (1 | 32)) {
 551                printd("ahci: smart fail %#lx\n", tfd);
 552                preg(rfis, 20);
 553                return -1;
 554        }
 555        if (rfis[5] == 0x4f && rfis[6] == 0xc2)
 556                return 1;
 557        return 0;
 558}
 559
 560static int ahciflushcache(struct aportc *pc)
 561{
 562        void *cfis;
 563
 564        cfis = cfissetup(pc);
 565        ahci_cfis_write8(cfis, 2, pc->pm->feat & Dllba ? 0xea : 0xe7);
 566        listsetup(pc, Lwrite);
 567        if (ahciwait(pc, 60000) == -1 ||
 568            ahci_port_read32(pc->p, PORT_TFD) & (1 | 32)) {
 569                printd("ahciflushcache: fail %#lx\n",
 570                       ahci_port_read32(pc->p, PORT_TFD));
 571                // preg(pc->m->fis.r, 20);
 572                return -1;
 573        }
 574        return 0;
 575}
 576
 577static uint16_t gbit16(void *a)
 578{
 579        unsigned char *i;
 580
 581        i = a;
 582        return i[1] << 8 | i[0];
 583}
 584
 585static uint32_t gbit32(void *a)
 586{
 587        uint32_t j;
 588        unsigned char *i;
 589
 590        i = a;
 591        j = i[3] << 24;
 592        j |= i[2] << 16;
 593        j |= i[1] << 8;
 594        j |= i[0];
 595        return j;
 596}
 597
 598static uint64_t gbit64(void *a)
 599{
 600        unsigned char *i;
 601
 602        i = a;
 603        return (uint64_t)gbit32(i + 4) << 32 | gbit32(a);
 604}
 605
 606static int ahciidentify0(struct aportc *pc, void *id, int atapi)
 607{
 608        void *cfis, *prdt;
 609        static unsigned char tab[] = {
 610            0xec,
 611            0xa1,
 612        };
 613
 614        cfis = cfissetup(pc);
 615        ahci_cfis_write8(cfis, 2, tab[atapi]);
 616        printd("ahci: %s: ATAPI=%d, cfis[2]: 0x%02x\n", __func__, atapi,
 617               ahci_cfis_read8(cfis, 2));
 618        printd("ahci: %s: CFIS physical address=0x%08x:0x%08x\n", __func__,
 619               paddr_high32(cfis), paddr_low32(cfis));
 620
 621        listsetup(pc, 1 << 16);
 622
 623        memset(id, 0, 0x200); /* Fill 512 bytes with 0's */
 624        prdt = pc->pm->ctab + ACTAB_PRDT;
 625        ahci_prdt_write32(prdt, APRDT_DBA, paddr_low32(id));
 626        ahci_prdt_write32(prdt, APRDT_DBAHI, paddr_high32(id));
 627        ahci_prdt_write32(prdt, APRDT_COUNT, 1 << 31 | (0x200 - 2) | 1);
 628        printd("ahci: %s: DBA  physical address=0x%08x:0x%08x\n", __func__,
 629               paddr_high32(id), paddr_low32(id));
 630
 631        return ahciwait(pc, 3 * 1000);
 632}
 633
 634static int64_t ahciidentify(struct aportc *pc, uint16_t *id)
 635{
 636        int i, sig;
 637        int64_t s;
 638        struct aportm *pm;
 639        int cnt;
 640
 641        pm = pc->pm;
 642        pm->feat = 0;
 643        pm->smart = 0;
 644        i = 0;
 645        sig = ahci_port_read32(pc->p, PORT_SIG) >> 16;
 646        if (sig == 0xeb14) {
 647                pm->feat |= Datapi;
 648                i = 1;
 649        }
 650        if (ahciidentify0(pc, id, i) == -1)
 651                return -1;
 652
 653#ifdef DEBUG
 654        printd("ahci: %s: ahciidentify0 return dump=\n\t", __func__);
 655        for (cnt = 0; cnt < 64; cnt++) {
 656                printd("0x%08x  ", id[cnt]);
 657                if (cnt % 4 == 3 && cnt != 63)
 658                        printd("\n\t");
 659        }
 660        printd("\n");
 661#endif
 662
 663        i = gbit16(id + 83) | gbit16(id + 86);
 664        if (i & (1 << 10)) {
 665                pm->feat |= Dllba;
 666                s = gbit64(id + 100);
 667        } else
 668                s = gbit32(id + 60);
 669
 670        if (pm->feat & Datapi) {
 671                i = gbit16(id + 0);
 672                if (i & 1)
 673                        pm->feat |= Datapi16;
 674        }
 675
 676        i = gbit16(id + 83);
 677        if ((i >> 14) == 1) {
 678                if (i & (1 << 3))
 679                        pm->feat |= Dpower;
 680                i = gbit16(id + 82);
 681                if (i & 1)
 682                        pm->feat |= Dsmart;
 683                if (i & (1 << 14))
 684                        pm->feat |= Dnop;
 685        }
 686        return s;
 687}
 688
 689#if 0
 690static int
 691ahciquiet(struct aport *a)
 692{
 693        uint32_t *p, i;
 694
 695        p = &a->cmd;
 696        *p &= ~Ast;
 697        for(i = 0; i < 500; i += 50){
 698                if((*p & Acr) == 0)
 699                        goto stop;
 700                asleep(50);
 701        }
 702        return -1;
 703stop:
 704        if((a->task & (ASdrq|ASbsy)) == 0){
 705                *p |= Ast;
 706                return 0;
 707        }
 708
 709        *p |= Aclo;
 710        for(i = 0; i < 500; i += 50){
 711                if((*p & Aclo) == 0)
 712                        goto stop1;
 713                asleep(50);
 714        }
 715        return -1;
 716stop1:
 717        /* extra check */
 718        printd("ahci: clo clear %#lx\n", a->task);
 719        if(a->task & ASbsy)
 720                return -1;
 721        *p |= Ast;
 722        return 0;
 723}
 724#endif
 725
 726#if 0
 727static int
 728ahcicomreset(struct aportc *pc)
 729{
 730        unsigned char *c;
 731
 732        printd("ahcicomreset\n");
 733        dreg("ahci: comreset ", pc->p);
 734        if(ahciquiet(pc->p) == -1){
 735                printd("ahciquiet failed\n");
 736                return -1;
 737        }
 738        dreg("comreset ", pc->p);
 739
 740        c = cfissetup(pc);
 741        ahci_cfis_write8(cfis, 1, 0);
 742        ahci_cfis_write8(cfis, 15, 1<<2);       /* srst */
 743        listsetup(pc, Lclear | Lreset);
 744        if(ahciwait(pc, 500) == -1){
 745                printd("ahcicomreset: first command failed\n");
 746                return -1;
 747        }
 748        microdelay(250);
 749        dreg("comreset ", pc->p);
 750
 751        c = cfissetup(pc);
 752        ahci_cfis_write8(cfis, 1, 0);
 753        listsetup(pc, Lwrite);
 754        if (ahciwait(pc, 150) == -1) {
 755                printd("ahcicomreset: second command failed\n");
 756                return -1;
 757        }
 758        dreg("comreset ", pc->p);
 759        return 0;
 760}
 761#endif
 762
 763static int ahciidle(void *port)
 764{
 765        uint32_t i, r, cmd;
 766
 767        cmd = ahci_port_read32(port, PORT_CMD);
 768        if ((cmd & Arun) == 0)
 769                return 0;
 770        cmd &= ~Ast;
 771        ahci_port_write32(port, PORT_CMD, cmd);
 772
 773        r = 0;
 774        for (i = 0; i < 500; i += 25) {
 775                if ((ahci_port_read32(port, PORT_CMD) & Acr) == 0)
 776                        goto stop;
 777                asleep(25);
 778        }
 779        r = -1;
 780stop:
 781        if ((ahci_port_read32(port, PORT_CMD) & Afre) == 0)
 782                return r;
 783
 784        clear_bit32(port, PORT_CMD, Afre);
 785        for (i = 0; i < 500; i += 25) {
 786                if ((ahci_port_read32(port, PORT_CMD) & Afre) == 0)
 787                        return 0;
 788                asleep(25);
 789        }
 790        return -1;
 791}
 792
 793/*
 794 * \xC2\xA7 6.2.2.1  first part; comreset handled by reset disk.
 795 *      - remainder is handled by configdisk.
 796 *      - ahcirecover is a quick recovery from a failed command.
 797 */
 798static int ahciswreset(struct aportc *pc)
 799{
 800        int i;
 801
 802        i = ahciidle(pc->p);
 803
 804        set_bit32(pc->p, PORT_CMD, Afre);
 805        if (i == -1)
 806                return -1;
 807        if (ahci_port_read32(pc->p, PORT_TFD) & (ASdrq | ASbsy))
 808                return -1;
 809        return 0;
 810}
 811
 812static int ahcirecover(struct aportc *pc)
 813{
 814        ahciswreset(pc);
 815
 816        set_bit32(pc->p, PORT_CMD, Ast);
 817        if (setudmamode(pc, 5) == -1)
 818                return -1;
 819        return 0;
 820}
 821
 822static void *malign(int size, int align)
 823{
 824        return kmalloc_align(size, MEM_WAIT, align);
 825}
 826
 827static void setupfis(struct afis *f)
 828{
 829        f->base = malign(0x100, 0x100); /* magic */
 830        f->d = f->base + 0;
 831        f->p = f->base + 0x20;
 832        f->r = f->base + 0x40;
 833        f->u = f->base + 0x60;
 834        f->devicebits = (uint32_t *)(f->base + 0x58);
 835}
 836
 837static void ahciwakeup(void *port)
 838{
 839        uint16_t s;
 840        uint32_t sctl;
 841
 842        s = ahci_port_read32(port, PORT_SSTS);
 843        if ((s & Intpm) != Intslumber && (s & Intpm) != Intpartpwr)
 844                return;
 845        if ((s & Devdet) != Devpresent) { /* not (device, no phy) */
 846                iprint("ahci: slumbering drive unwakable %#x\n", s);
 847                return;
 848        }
 849        ahci_port_write32(port, PORT_SCTL, 3 * Aipm | 0 * Aspd | Adet);
 850        udelay(1000 * 1000);
 851        clear_bit32(port, PORT_SCTL, 7);
 852        //      iprint("ahci: wake %#x -> %#x\n", s, p->sstatus);
 853}
 854
 855static int ahciconfigdrive(struct drive *d)
 856{
 857        char *name;
 858        void *hba, *port;
 859        struct aportm *pm;
 860        uint32_t cap, sstatus;
 861
 862        hba = d->ctlr->hba;
 863        port = d->portc.p;
 864        pm = d->portc.pm;
 865        if (pm->list == 0) {
 866                setupfis(&pm->fis);
 867                pm->list = malign(ALIST_SIZE, 1024);
 868                pm->ctab = malign(ACTAB_PRDT + APRDT_SIZE, 128);
 869        }
 870
 871        if (d->unit)
 872                name = d->unit->sdperm.name;
 873        else
 874                name = NULL;
 875        sstatus = ahci_port_read32(port, PORT_SSTS);
 876        cap = ahci_hba_read32(hba, HBA_CAP);
 877        if (sstatus & (Devphycomm | Devpresent) && cap & Hsss) {
 878                /* device connected & staggered spin-up */
 879                printd("ahci: configdrive: %s: spinning up ... [%#lx]\n", name,
 880                       sstatus);
 881                set_bit32(port, PORT_CMD, Apod | Asud);
 882                asleep(1400);
 883        }
 884
 885        ahci_port_write32(port, PORT_SERR, SerrAll);
 886
 887        ahci_port_write32(port, PORT_CLB, paddr_low32(pm->list));
 888        ahci_port_write32(port, PORT_CLBU, paddr_high32(pm->list));
 889        printd("ahci: %s: PORT_CLB physical address=0x%08x:0x%08x\n", __func__,
 890               paddr_high32(pm->list), paddr_low32(pm->list));
 891        ahci_port_write32(port, PORT_FB, paddr_low32(pm->fis.base));
 892        ahci_port_write32(port, PORT_FBU, paddr_high32(pm->fis.base));
 893        printd("ahci: %s: PORT_FB  physical address=0x%08x:0x%08x\n", __func__,
 894               paddr_high32(pm->fis.base), paddr_low32(pm->fis.base));
 895
 896        set_bit32(port, PORT_CMD, Afre | Ast);
 897
 898        /* drive coming up in slumbering? */
 899        sstatus = ahci_port_read32(port, PORT_SSTS);
 900        if ((sstatus & Devdet) == Devpresent &&
 901            ((sstatus & Intpm) == Intslumber ||
 902             (sstatus & Intpm) == Intpartpwr))
 903                ahciwakeup(port);
 904
 905        /* "disable power management" sequence from book. */
 906        ahci_port_write32(port, PORT_SCTL,
 907                          (3 * Aipm) | (d->mode * Aspd) | (0 * Adet));
 908        clear_bit32(port, PORT_CMD, Aalpe);
 909        ahci_port_write32(port, PORT_IE, IEM);
 910
 911        return 0;
 912}
 913
 914static void ahcienable(void *hba)
 915{
 916        set_bit32(hba, HBA_GHC, Hie);
 917}
 918
 919static void ahcidisable(void *hba)
 920{
 921        clear_bit32(hba, HBA_GHC, Hie);
 922}
 923
 924static int countbits(uint32_t u)
 925{
 926        int n;
 927
 928        n = 0;
 929        for (; u != 0; u >>= 1)
 930                if (u & 1)
 931                        n++;
 932        return n;
 933}
 934
 935static int ahciconf(struct ctlr *ctlr)
 936{
 937        void *hba;
 938        uint32_t cap;
 939
 940        hba = ctlr->hba = ctlr->mmio;
 941        cap = ahci_hba_read32(hba, HBA_CAP);
 942
 943        if ((cap & Hsam) == 0)
 944                set_bit32(hba, HBA_GHC, Hae);
 945
 946        printd("#S/sd%c: type %s port %#p: sss %ld ncs %ld coal %ld "
 947               "%ld ports, led %ld clo %ld ems %ld\n",
 948               ctlr->sdev->idno, tname[ctlr->type], hba, (cap >> 27) & 1,
 949               (cap >> 8) & 0x1f, (cap >> 7) & 1, (cap & 0x1f) + 1,
 950               (cap >> 25) & 1, (cap >> 24) & 1, (cap >> 6) & 1);
 951        return countbits(ahci_hba_read32(hba, HBA_PI));
 952}
 953
 954#if 0
 955static int
 956ahcihbareset(void *hba)
 957{
 958        int wait;
 959
 960        h->ghc |= 1;
 961        for(wait = 0; wait < 1000; wait += 100){
 962                if(h->ghc == 0)
 963                        return 0;
 964                delay(100);
 965        }
 966        return -1;
 967}
 968#endif
 969
 970static void idmove(char *p, uint16_t *a, int n)
 971{
 972        int i;
 973        char *op, *e;
 974
 975        op = p;
 976        for (i = 0; i < n / 2; i++) {
 977                *p++ = a[i] >> 8;
 978                *p++ = a[i];
 979        }
 980        *p = 0;
 981        while (p > op && *--p == ' ')
 982                *p = 0;
 983        e = p;
 984        for (p = op; *p == ' '; p++)
 985                ;
 986        memmove(op, p, n - (e - p));
 987}
 988
 989static int identify(struct drive *d)
 990{
 991        uint16_t *id;
 992        int64_t osectors, s;
 993        unsigned char oserial[21];
 994        struct sdunit *u;
 995
 996        if (d->info == NULL) {
 997                d->infosz = 512 * sizeof(uint16_t);
 998                d->info = kzmalloc(d->infosz, 0);
 999                printd("ahci: %s: HBA physical address (hack): 0x%08x:0x%08x\n",
1000                       __func__, paddr_high32(d->info), paddr_low32(d->info));
1001        }
1002        if (d->info == NULL) {
1003                d->info = d->tinyinfo;
1004                d->infosz = sizeof d->tinyinfo;
1005        }
1006        id = d->info;
1007        s = ahciidentify(&d->portc, id);
1008        if (s == -1) {
1009                d->state = Derror;
1010                return -1;
1011        }
1012        osectors = d->sectors;
1013        memmove(oserial, d->serial, sizeof d->serial);
1014
1015        u = d->unit;
1016        d->sectors = s;
1017        d->secsize = u->secsize;
1018        if (d->secsize == 0)
1019                d->secsize = 512; /* default */
1020        d->smartrs = 0;
1021
1022        idmove(d->serial, id + 10, 20);
1023        idmove(d->firmware, id + 23, 8);
1024        idmove(d->model, id + 27, 40);
1025
1026        memset(u->inquiry, 0, sizeof u->inquiry);
1027        u->inquiry[2] = 2;
1028        u->inquiry[3] = 2;
1029        u->inquiry[4] = sizeof u->inquiry - 4;
1030        memmove(u->inquiry + 8, d->model, 40);
1031
1032        if (osectors != s || memcmp(oserial, d->serial, sizeof oserial) != 0) {
1033                d->mediachange = 1;
1034                u->sectors = 0;
1035        }
1036        return 0;
1037}
1038
1039static void clearci(void *p)
1040{
1041        uint32_t cmd;
1042
1043        cmd = ahci_port_read32(p, PORT_CMD);
1044        if (cmd & Ast) {
1045                clear_bit32(p, PORT_CMD, Ast);
1046                set_bit32(p, PORT_CMD, Ast);
1047        }
1048}
1049
1050static void updatedrive(struct drive *d)
1051{
1052        uint32_t cause, serr, task, sstatus, ie, s0, pr, ewake;
1053        char *name;
1054        void *port;
1055        static uint32_t last;
1056
1057        pr = 1;
1058        ewake = 0;
1059        port = d->port;
1060        cause = ahci_port_read32(port, PORT_IS);
1061        serr = ahci_port_read32(port, PORT_SERR);
1062        ahci_port_write32(port, PORT_IS, cause);
1063        name = "??";
1064        if (d->unit && d->unit->sdperm.name)
1065                name = d->unit->sdperm.name;
1066
1067        if (ahci_port_read32(port, PORT_CI) == 0) {
1068                d->portm.flag |= Fdone;
1069                rendez_wakeup(&d->portm.Rendez);
1070                pr = 0;
1071        } else if (cause & Adps)
1072                pr = 0;
1073        if (cause & Ifatal) {
1074                ewake = 1;
1075                printd("ahci: updatedrive: %s: fatal\n", name);
1076        }
1077        task = ahci_port_read32(port, PORT_TFD);
1078        if (cause & Adhrs) {
1079                if (task & (1 << 5 | 1)) {
1080                        printd(
1081                            "ahci: %s: Adhrs cause %#lx serr %#lx task %#lx\n",
1082                            name, cause, serr, task);
1083                        d->portm.flag |= Ferror;
1084                        ewake = 1;
1085                }
1086                pr = 0;
1087        }
1088        sstatus = ahci_port_read32(port, PORT_SSTS);
1089        if (task & 1 && last != cause)
1090                printd("%s: err ca %#lx serr %#lx task %#lx sstat %#lx\n", name,
1091                       cause, serr, task, sstatus);
1092        if (pr)
1093                printd("%s: upd %#lx ta %#lx\n", name, cause, task);
1094
1095        if (cause & (Aprcs | Aifs)) {
1096                s0 = d->state;
1097                switch (sstatus & Devdet) {
1098                case 0: /* no device */
1099                        d->state = Dmissing;
1100                        break;
1101                case Devpresent: /* device but no phy comm. */
1102                        if ((sstatus & Intpm) == Intslumber ||
1103                            (sstatus & Intpm) == Intpartpwr)
1104                                d->state = Dnew; /* slumbering */
1105                        else
1106                                d->state = Derror;
1107                        break;
1108                case Devpresent | Devphycomm:
1109                        /* power mgnt crap for surprise removal */
1110                        set_bit32(port, PORT_IE,
1111                                  Aprcs | Apcs); /* is this required? */
1112                        d->state = Dreset;
1113                        break;
1114                case Devphyoffline:
1115                        d->state = Doffline;
1116                        break;
1117                }
1118                printd("%s: %s \xE2\x86\x92 %s [Apcrs] %#lx\n", name, diskstates[s0],
1119                       diskstates[d->state], sstatus);
1120                /* print pulled message here. */
1121                if (s0 == Dready && d->state != Dready)
1122                        iprintd("%s: pulled\n", name); /* wtf? */
1123                if (d->state != Dready)
1124                        d->portm.flag |= Ferror;
1125                ewake = 1;
1126        }
1127        ahci_port_write32(port, PORT_SERR, serr);
1128        if (ewake) {
1129                clearci(port);
1130                rendez_wakeup(&d->portm.Rendez);
1131        }
1132        last = cause;
1133}
1134
1135static void pstatus(struct drive *d, uint32_t s)
1136{
1137        /*
1138         * s is masked with Devdet.
1139         *
1140         * bogus code because the first interrupt is currently dropped.
1141         * likely my fault.  serror may be cleared at the wrong time.
1142         */
1143        switch (s) {
1144        case 0: /* no device */
1145                d->state = Dmissing;
1146                break;
1147        case Devpresent: /* device but no phy. comm. */
1148                break;
1149        case Devphycomm: /* should this be missing?  need testcase. */
1150                printd("ahci: pstatus 2\n");
1151        /* fallthrough */
1152        case Devpresent | Devphycomm:
1153                d->wait = 0;
1154                d->state = Dnew;
1155                break;
1156        case Devphyoffline:
1157                d->state = Doffline;
1158                break;
1159        case Devphyoffline | Devphycomm: /* does this make sense? */
1160                d->state = Dnew;
1161                break;
1162        }
1163}
1164
1165static int configdrive(struct drive *d)
1166{
1167        if (ahciconfigdrive(d) == -1)
1168                return -1;
1169        spin_lock_irqsave(&d->Lock);
1170        pstatus(d, ahci_port_read32(d->port, PORT_SSTS) & Devdet);
1171        spin_unlock_irqsave(&d->Lock);
1172        return 0;
1173}
1174
1175static void setstate(struct drive *d, int state)
1176{
1177        spin_lock_irqsave(&d->Lock);
1178        d->state = state;
1179        spin_unlock_irqsave(&d->Lock);
1180}
1181
1182static void resetdisk(struct drive *d)
1183{
1184        unsigned int state, det, cmd, stat;
1185        void *port;
1186
1187        port = d->port;
1188        det = ahci_port_read32(port, PORT_SCTL) & 7;
1189        stat = ahci_port_read32(port, PORT_SSTS) & Devdet;
1190        cmd = ahci_port_read32(port, PORT_CMD);
1191        state = (cmd >> 28) & 0xf;
1192        printd("ahci: resetdisk: icc %#x  det %d sdet %d\n", state, det, stat);
1193
1194        spin_lock_irqsave(&d->Lock);
1195        state = d->state;
1196        if (d->state != Dready || d->state != Dnew)
1197                d->portm.flag |= Ferror;
1198        clearci(port); /* satisfy sleep condition. */
1199        rendez_wakeup(&d->portm.Rendez);
1200        if (stat != (Devpresent | Devphycomm)) {
1201                /* device absent or phy not communicating */
1202                d->state = Dportreset;
1203                spin_unlock_irqsave(&d->Lock);
1204                return;
1205        }
1206        d->state = Derror;
1207        spin_unlock_irqsave(&d->Lock);
1208
1209        qlock(&d->portm.ql);
1210        if (cmd & Ast && ahciswreset(&d->portc) == -1)
1211                setstate(d, Dportreset); /* get a bigger stick. */
1212        else {
1213                setstate(d, Dmissing);
1214                configdrive(d);
1215        }
1216        printd("ahci: %s: resetdisk: %s \xE2\x86\x92 %s\n",
1217               (d->unit ? d->unit->sdperm.name : NULL), diskstates[state],
1218               diskstates[d->state]);
1219        qunlock(&d->portm.ql);
1220}
1221
1222static int newdrive(struct drive *d)
1223{
1224        char *name;
1225        struct aportc *pc;
1226        struct aportm *pm;
1227
1228        pc = &d->portc;
1229        pm = &d->portm;
1230
1231        name = d->unit->sdperm.name;
1232        if (name == 0)
1233                name = "??";
1234
1235        if (ahci_port_read32(d->port, PORT_TFD) == 0x80)
1236                return -1;
1237        qlock(&pc->pm->ql);
1238        if (setudmamode(pc, 5) == -1) {
1239                printd("%s: can't set udma mode\n", name);
1240                goto lose;
1241        }
1242        if (identify(d) == -1) {
1243                printd("%s: identify failure\n", name);
1244                goto lose;
1245        }
1246        if (pm->feat & Dpower && setfeatures(pc, 0x85) == -1) {
1247                pm->feat &= ~Dpower;
1248                if (ahcirecover(pc) == -1)
1249                        goto lose;
1250        }
1251        setstate(d, Dready);
1252        qunlock(&pc->pm->ql);
1253
1254        iprintd("%s: %sLBA %llu sectors: %s %s %s %s\n", d->unit->sdperm.name,
1255                (pm->feat & Dllba ? "L" : ""), d->sectors, d->model,
1256                d->firmware, d->serial, d->mediachange ? "[mediachange]" : "");
1257        return 0;
1258
1259lose:
1260        iprintd("%s: can't be initialized\n", d->unit->sdperm.name);
1261        setstate(d, Dnull);
1262        qunlock(&pc->pm->ql);
1263        return -1;
1264}
1265
1266static void westerndigitalhung(struct drive *d)
1267{
1268        uint32_t task, ci;
1269
1270        task = ahci_port_read32(d->port, PORT_TFD);
1271        ci = ahci_port_read32(d->port, PORT_CI);
1272
1273        if ((d->portm.feat & Datapi) == 0 && d->active &&
1274            (ms() - d->intick) > 5000) {
1275                printd("%s: drive hung; resetting [%#lx] ci %#lx\n",
1276                       d->unit->sdperm.name, task, ci);
1277                d->state = Dreset;
1278        }
1279}
1280
1281static uint16_t olds[NCtlr * NCtlrdrv];
1282
1283static int doportreset(struct drive *d)
1284{
1285        int i;
1286
1287        i = -1;
1288        qlock(&d->portm.ql);
1289        if (ahciportreset(&d->portc) == -1)
1290                printd("ahci: doportreset: fails\n") else i = 0;
1291        qunlock(&d->portm.ql);
1292        printd("ahci: doportreset: portreset \xE2\x86\x92 %s  [task %#lx]\n",
1293               diskstates[d->state], ahci_port_read32(d->port, PORT_TFD));
1294        return i;
1295}
1296
1297/* drive must be locked */
1298static void statechange(struct drive *d)
1299{
1300        switch (d->state) {
1301        case Dnull:
1302        case Doffline:
1303                if (d->unit->sectors != 0) {
1304                        d->sectors = 0;
1305                        d->mediachange = 1;
1306                }
1307        /* fallthrough */
1308        case Dready:
1309                d->wait = 0;
1310                break;
1311        }
1312}
1313
1314static void checkdrive(struct drive *d, int i)
1315{
1316        uint16_t s;
1317        uint32_t task;
1318        char *name;
1319
1320        if (d == NULL) {
1321                printd("checkdrive: NULL d\n");
1322                return;
1323        }
1324        spin_lock_irqsave(&d->Lock);
1325        if (d->unit == NULL || d->port == NULL) {
1326                if (0)
1327                        printk("checkdrive: nil d->%s\n",
1328                               d->unit == NULL ? "unit" : "port");
1329                spin_unlock_irqsave(&d->Lock);
1330                return;
1331        }
1332        name = d->unit->sdperm.name;
1333        s = ahci_port_read32(d->port, PORT_SSTS);
1334        if (s)
1335                d->lastseen = ms();
1336        if (s != olds[i]) {
1337                printd("%s: status: %06#x -> %06#x: %s\n", name, olds[i], s,
1338                       diskstates[d->state]);
1339                olds[i] = s;
1340                d->wait = 0;
1341        }
1342        westerndigitalhung(d);
1343
1344        switch (d->state) {
1345        case Dnull:
1346        case Dready:
1347                break;
1348        case Dmissing:
1349        case Dnew:
1350                switch (s & (Intactive | Devdet)) {
1351                case Devpresent: /* no device (pm), device but no phy. comm. */
1352                        ahciwakeup(d->port);
1353                /* fall through */
1354                case 0: /* no device */
1355                        break;
1356                default:
1357                        printd("%s: unknown status %06#x\n", name, s);
1358                /* fall through */
1359                case Intactive: /* active, no device */
1360                        if (++d->wait & Mphywait)
1361                                break;
1362                reset:
1363                        if (++d->mode > DMsataii)
1364                                d->mode = 0;
1365                        if (d->mode == DMsatai) { /* we tried everything */
1366                                d->state = Dportreset;
1367                                goto portreset;
1368                        }
1369                        printd("%s: reset; new mode %s\n", name,
1370                               modename[d->mode]);
1371                        spin_unlock_irqsave(&d->Lock);
1372                        resetdisk(d);
1373                        spin_lock_irqsave(&d->Lock);
1374                        break;
1375                case Intactive | Devphycomm | Devpresent:
1376                        task = ahci_port_read32(d->port, PORT_TFD);
1377                        if ((++d->wait & Midwait) == 0) {
1378                                printd("%s: slow reset %06#x task=%#lx; %d\n",
1379                                       name, s, task, d->wait);
1380                                goto reset;
1381                        }
1382                        s = (unsigned char)task;
1383                        if (s == 0x7f ||
1384                            ((ahci_port_read32(d->port, PORT_SIG) >> 16) !=
1385                                 0xeb14 &&
1386                             (s & ~0x17) != (1 << 6)))
1387                                break;
1388                        spin_unlock_irqsave(&d->Lock);
1389                        newdrive(d);
1390                        spin_lock_irqsave(&d->Lock);
1391                        break;
1392                }
1393                break;
1394        case Doffline:
1395                if (d->wait++ & Mcomrwait)
1396                        break;
1397        /* fallthrough */
1398        case Derror:
1399        case Dreset:
1400                printd("%s: reset [%s]: mode %d; status %06#x\n", name,
1401                       diskstates[d->state], d->mode, s);
1402                spin_unlock_irqsave(&d->Lock);
1403                resetdisk(d);
1404                spin_lock_irqsave(&d->Lock);
1405                break;
1406        case Dportreset:
1407        portreset:
1408                if (d->wait++ & 0xff && (s & Intactive) == 0)
1409                        break;
1410                /* device is active */
1411                printd("%s: portreset [%s]: mode %d; status %06#x\n", name,
1412                       diskstates[d->state], d->mode, s);
1413                d->portm.flag |= Ferror;
1414                clearci(d->port);
1415                rendez_wakeup(&d->portm.Rendez);
1416                if ((s & Devdet) == 0) { /* no device */
1417                        d->state = Dmissing;
1418                        break;
1419                }
1420                spin_unlock_irqsave(&d->Lock);
1421                doportreset(d);
1422                spin_lock_irqsave(&d->Lock);
1423                break;
1424        }
1425        statechange(d);
1426        spin_unlock_irqsave(&d->Lock);
1427}
1428
1429static void satakproc(void *v)
1430{
1431        int i;
1432        for (;;) {
1433                kthread_usleep(Nms * 1000);
1434                for (i = 0; i < niadrive; i++)
1435                        if (iadrive[i] != NULL)
1436                                checkdrive(iadrive[i], i);
1437        }
1438}
1439
1440static void isctlrjabbering(struct ctlr *c, uint32_t cause)
1441{
1442        uint32_t now;
1443
1444        now = ms();
1445        if (now > c->lastintr0) {
1446                c->intrs = 0;
1447                c->lastintr0 = now;
1448        }
1449        if (++c->intrs > Maxintrspertick) {
1450                iprint("sdiahci: %lu intrs per tick for no serviced "
1451                       "drive; cause %#lx mport %d\n",
1452                       c->intrs, cause, c->mport);
1453                c->intrs = 0;
1454        }
1455}
1456
1457static void isdrivejabbering(struct drive *d)
1458{
1459        uint32_t now = ms();
1460
1461        if (now > d->lastintr0) {
1462                d->intrs = 0;
1463                d->lastintr0 = now;
1464        }
1465        if (++d->intrs > Maxintrspertick) {
1466                iprint("sdiahci: %lu interrupts per tick for %s\n", d->intrs,
1467                       d->unit->sdperm.name);
1468                d->intrs = 0;
1469        }
1470}
1471
1472static void iainterrupt(struct hw_trapframe *unused_hw_trapframe, void *a)
1473{
1474        int i;
1475        uint32_t cause, mask, p_is, h_is, pi;
1476        struct ctlr *c;
1477        struct drive *d;
1478
1479        c = a;
1480        spin_lock_irqsave(&c->Lock);
1481        cause = ahci_hba_read32(c->hba, HBA_ISR);
1482        if (cause == 0) {
1483                isctlrjabbering(c, cause);
1484                // iprint("sdiahci: interrupt for no drive\n");
1485                spin_unlock_irqsave(&c->Lock);
1486                return;
1487        }
1488        for (i = 0; cause && i <= c->mport; i++) {
1489                mask = 1 << i;
1490                if ((cause & mask) == 0)
1491                        continue;
1492                d = c->rawdrive + i;
1493                spin_lock_irqsave(&d->Lock);
1494                isdrivejabbering(d);
1495                p_is = ahci_port_read32(d->port, PORT_IS);
1496                pi = ahci_hba_read32(c->hba, HBA_PI);
1497                if (p_is && pi & mask)
1498                        updatedrive(d);
1499                ahci_hba_write32(c->hba, HBA_ISR, mask);
1500                spin_unlock_irqsave(&d->Lock);
1501
1502                cause &= ~mask;
1503        }
1504        if (cause) {
1505                isctlrjabbering(c, cause);
1506                iprint("sdiahci: intr cause unserviced: %#lx\n", cause);
1507        }
1508        spin_unlock_irqsave(&c->Lock);
1509}
1510
1511/* checkdrive, called from satakproc, will prod the drive while we wait */
1512static void awaitspinup(struct drive *d)
1513{
1514        int ms;
1515        uint16_t s;
1516        char *name;
1517
1518        spin_lock_irqsave(&d->Lock);
1519        if (d->unit == NULL || d->port == NULL) {
1520                panic("awaitspinup: NULL d->unit or d->port");
1521                spin_unlock_irqsave(&d->Lock);
1522                return;
1523        }
1524        name = (d->unit ? d->unit->sdperm.name : NULL);
1525        s = ahci_port_read32(d->port, PORT_SSTS);
1526        if (!(s & Devpresent)) { /* never going to be ready */
1527                printd("awaitspinup: %s absent, not waiting\n", name);
1528                spin_unlock_irqsave(&d->Lock);
1529                return;
1530        }
1531
1532        for (ms = 20000; ms > 0; ms -= 50)
1533                switch (d->state) {
1534                case Dnull:
1535                        /* absent; done */
1536                        spin_unlock_irqsave(&d->Lock);
1537                        printd("awaitspinup: %s in null state\n", name);
1538                        return;
1539                case Dready:
1540                case Dnew:
1541                        if (d->sectors || d->mediachange) {
1542                                /* ready to use; done */
1543                                spin_unlock_irqsave(&d->Lock);
1544                                printd("awaitspinup: %s ready!\n", name);
1545                                return;
1546                        }
1547                /* fall through */
1548                default:
1549                case Dmissing: /* normal waiting states */
1550                case Dreset:
1551                case Doffline: /* transitional states */
1552                case Derror:
1553                case Dportreset:
1554                        spin_unlock_irqsave(&d->Lock);
1555                        asleep(50);
1556                        spin_lock_irqsave(&d->Lock);
1557                        break;
1558                }
1559        printd("awaitspinup: %s didn't spin up after 20 seconds\n", name);
1560        spin_unlock_irqsave(&d->Lock);
1561}
1562
1563static int iaverify(struct sdunit *u)
1564{
1565        struct ctlr *c;
1566        struct drive *d;
1567
1568        c = u->dev->ctlr;
1569        d = c->drive[u->subno];
1570        spin_lock_irqsave(&c->Lock);
1571        spin_lock_irqsave(&d->Lock);
1572        d->unit = u;
1573        spin_unlock_irqsave(&d->Lock);
1574        spin_unlock_irqsave(&c->Lock);
1575        checkdrive(d, d->driveno); /* c->d0 + d->driveno */
1576
1577        /*
1578         * hang around until disks are spun up and thus available as
1579         * nvram, dos file systems, etc.  you wouldn't expect it, but
1580         * the intel 330 ssd takes a while to `spin up'.
1581         */
1582        awaitspinup(d);
1583        return 1;
1584}
1585
1586static int iaenable(struct sdev *s)
1587{
1588        char name[32];
1589        struct ctlr *c;
1590        static int once;
1591
1592        c = s->ctlr;
1593        spin_lock_irqsave(&c->Lock);
1594        if (!c->enabled) {
1595                if (once == 0) {
1596                        once = 1;
1597                        ktask("ahci", satakproc, 0);
1598                }
1599                if (c->ndrive == 0)
1600                        panic("iaenable: zero s->ctlr->ndrive");
1601                pci_set_bus_master(c->pci);
1602                snprintf(name, sizeof(name), "%s (%s)", s->name, s->ifc->name);
1603                /*c->vector = intrenable(c->pci->intl, iainterrupt, c,
1604                 *c->pci->tbdf, name);*/
1605                /* what do we do about the arg? */
1606                register_irq(c->pci->irqline, iainterrupt, c,
1607                             pci_to_tbdf(c->pci));
1608                /* supposed to squelch leftover interrupts here. */
1609                ahcienable(c->hba);
1610                c->enabled = 1;
1611        }
1612        spin_unlock_irqsave(&c->Lock);
1613        return 1;
1614}
1615
1616static int iadisable(struct sdev *s)
1617{
1618        char name[32];
1619        struct ctlr *c;
1620
1621        c = s->ctlr;
1622        spin_lock_irqsave(&c->Lock);
1623        ahcidisable(c->hba);
1624        snprintf(name, sizeof(name), "%s (%s)", s->name, s->ifc->name);
1625        // TODO: what to do here?
1626        // intrdisable(c->vector);
1627        c->enabled = 0;
1628        spin_unlock_irqsave(&c->Lock);
1629        return 1;
1630}
1631
1632static int iaonline(struct sdunit *unit)
1633{
1634        int r;
1635        struct ctlr *c;
1636        struct drive *d;
1637
1638        c = unit->dev->ctlr;
1639        d = c->drive[unit->subno];
1640        r = 0;
1641
1642        if ((d->portm.feat & Datapi) && d->mediachange) {
1643                r = scsionline(unit);
1644                if (r > 0)
1645                        d->mediachange = 0;
1646                return r;
1647        }
1648
1649        spin_lock_irqsave(&d->Lock);
1650        if (d->mediachange) {
1651                r = 2;
1652                d->mediachange = 0;
1653                /* devsd resets this after online is called; why? */
1654                unit->sectors = d->sectors;
1655                unit->secsize = 512; /* default size */
1656        } else if (d->state == Dready)
1657                r = 1;
1658        spin_unlock_irqsave(&d->Lock);
1659        return r;
1660}
1661
1662/* returns locked list! */
1663static void *ahcibuild(struct drive *d, unsigned char *cmd, void *data, int n,
1664                       int64_t lba)
1665{
1666        void *cfis, *list, *prdt, *ctab;
1667        unsigned char acmd, dir, llba, c7;
1668        struct aportm *pm;
1669        uint32_t flags;
1670        static unsigned char tab[2][2] = {
1671            {0xc8, 0x25},
1672            {0xca, 0x35},
1673        };
1674
1675        pm = &d->portm;
1676        dir = *cmd != 0x28;
1677        llba = pm->feat & Dllba ? 1 : 0;
1678        acmd = tab[dir][llba];
1679        qlock(&pm->ql);
1680        list = pm->list;
1681        ctab = pm->ctab;
1682        cfis = ctab;
1683
1684        ahci_cfis_write8(cfis, 0, 0x27);
1685        ahci_cfis_write8(cfis, 1, 0x80);
1686        ahci_cfis_write8(cfis, 2, acmd);
1687        ahci_cfis_write8(cfis, 3, 0);
1688
1689        ahci_cfis_write8(cfis, 4, lba);       /* sector                 lba low
1690                                                 7:0 */
1691        ahci_cfis_write8(cfis, 5, lba >> 8);  /* cylinder low   lba mid 15:8 */
1692        ahci_cfis_write8(cfis, 6, lba >> 16); /* cylinder hi    lba hi  23:16 */
1693        c7 = Obs | 0x40;                      /* 0x40 == lba */
1694        if (llba == 0)
1695                c7 |= (lba >> 24) & 7;
1696        ahci_cfis_write8(cfis, 7, c7);
1697
1698        ahci_cfis_write8(
1699            cfis, 8,
1700            lba >> 24); /* sector (exp)                 lba     31:24 */
1701        ahci_cfis_write8(cfis, 9,
1702                         lba >> 32); /* cylinder low (exp)      lba     39:32 */
1703        ahci_cfis_write8(
1704            cfis, 10, lba >> 48);      /* cylinder hi (exp)             lba     48:40 */
1705        ahci_cfis_write8(cfis, 11, 0); /* features (exp); */
1706
1707        ahci_cfis_write8(cfis, 12, n);      /* sector count */
1708        ahci_cfis_write8(cfis, 13, n >> 8); /* sector count (exp) */
1709        ahci_cfis_write8(cfis, 14, 0);      /* r */
1710        ahci_cfis_write8(cfis, 15, 0);      /* control */
1711
1712        ahci_cfis_write8(cfis, 16, 0);
1713        ahci_cfis_write8(cfis, 17, 0);
1714        ahci_cfis_write8(cfis, 18, 0);
1715        ahci_cfis_write8(cfis, 19, 0);
1716
1717        flags = 1 << 16 | Lpref | 0x5; /* Lpref ?? */
1718        if (dir == Write)
1719                flags |= Lwrite;
1720        ahci_list_write32(list, ALIST_FLAGS, flags);
1721        ahci_list_write32(list, ALIST_LEN, 0);
1722        ahci_list_write32(list, ALIST_CTAB, paddr_low32(ctab));
1723        ahci_list_write32(list, ALIST_CTABHI, paddr_high32(ctab));
1724
1725        prdt = ctab + ACTAB_PRDT;
1726        ahci_prdt_write32(prdt, APRDT_DBA, paddr_low32(data));
1727        ahci_prdt_write32(prdt, APRDT_DBAHI, paddr_high32(data));
1728        if (d->unit == NULL)
1729                panic("ahcibuild: NULL d->unit");
1730        ahci_prdt_write32(prdt, APRDT_COUNT,
1731                          1 << 31 | (d->unit->secsize * n - 2) | 1);
1732
1733        return list;
1734}
1735
1736static void *ahcibuildpkt(struct aportm *pm, struct sdreq *r, void *data, int n)
1737{
1738        int fill, len, i;
1739        void *cfis, *list, *ctab, *prdt;
1740        uint32_t flags;
1741
1742        qlock(&pm->ql);
1743        list = pm->list;
1744        ctab = pm->ctab;
1745        cfis = ctab;
1746
1747        fill = pm->feat & Datapi16 ? 16 : 12;
1748        if ((len = r->clen) > fill)
1749                len = fill;
1750        memmove(ctab + ACTAB_ATAPI, r->cmd, len);
1751        memset(ctab + ACTAB_ATAPI + len, 0, fill - len);
1752
1753        ahci_cfis_write8(cfis, 0, 0x27);
1754        ahci_cfis_write8(cfis, 1, 0x80);
1755        ahci_cfis_write8(cfis, 2, 0xa0);
1756        if (n != 0)
1757                ahci_cfis_write8(cfis, 3, 1); /* dma */
1758        else
1759                ahci_cfis_write8(cfis, 3, 0); /* features (exp); */
1760
1761        ahci_cfis_write8(cfis, 4, 0); /* sector                 lba low 7:0 */
1762        ahci_cfis_write8(cfis, 5, n); /* cylinder low           lba mid 15:8 */
1763        ahci_cfis_write8(cfis, 6,
1764                         n >> 8); /* cylinder hi                lba hi  23:16 */
1765        ahci_cfis_write8(cfis, 7, Obs);
1766
1767        for (i = 0; i < 12; i++)
1768                ahci_cfis_write8(cfis, 8 + i, 0);
1769
1770        flags = 1 << 16 | Lpref | Latapi | 0x5;
1771        if (r->write != 0 && data)
1772                flags |= Lwrite;
1773        ahci_list_write32(list, ALIST_FLAGS, flags);
1774        ahci_list_write32(list, ALIST_LEN, 0);
1775        ahci_list_write32(list, ALIST_CTAB, paddr_low32(ctab));
1776        ahci_list_write32(list, ALIST_CTABHI, paddr_high32(ctab));
1777        printd("ahci: %s: LIST->CTAB physical address=0x%08x:0x%08x\n",
1778               __func__, paddr_high32(ctab), paddr_low32(ctab));
1779
1780        if (data == 0)
1781                return list;
1782
1783        prdt = ctab + ACTAB_PRDT;
1784        ahci_prdt_write32(prdt, APRDT_DBA, paddr_low32(data));
1785        ahci_prdt_write32(prdt, APRDT_DBAHI, paddr_high32(data));
1786        ahci_prdt_write32(prdt, APRDT_COUNT, 1 << 31 | (n - 2) | 1);
1787        printd("ahci: %s: PRDT->DBA  physical address=0x%08x:0x%08x\n",
1788               __func__, paddr_high32(data), paddr_low32(data));
1789
1790        return list;
1791}
1792
1793static int waitready(struct drive *d)
1794{
1795        uint32_t s, i, delta;
1796
1797        for (i = 0; i < 15000; i += 250) {
1798                if (d->state == Dreset || d->state == Dportreset ||
1799                    d->state == Dnew)
1800                        return 1;
1801                delta = ms() - d->lastseen;
1802                if (d->state == Dnull || delta > 10 * 1000)
1803                        return -1;
1804                spin_lock_irqsave(&d->Lock);
1805                s = ahci_port_read32(d->port, PORT_SSTS);
1806                spin_unlock_irqsave(&d->Lock);
1807                if ((s & Intpm) == 0 && delta > 1500)
1808                        return -1; /* no detect */
1809                if (d->state == Dready &&
1810                    (s & Devdet) == (Devphycomm | Devpresent))
1811                        return 0; /* ready, present & phy. comm. */
1812                esleep(250);
1813        }
1814        printd("%s: not responding; offline\n", d->unit->sdperm.name);
1815        setstate(d, Doffline);
1816        return -1;
1817}
1818
1819static int lockready(struct drive *d)
1820{
1821        int i;
1822
1823        qlock(&d->portm.ql);
1824        while ((i = waitready(d)) == 1) { /* could wait forever? */
1825                qunlock(&d->portm.ql);
1826                esleep(1);
1827                qlock(&d->portm.ql);
1828        }
1829        return i;
1830}
1831
1832static int flushcache(struct drive *d)
1833{
1834        int i;
1835
1836        i = -1;
1837        if (lockready(d) == 0)
1838                i = ahciflushcache(&d->portc);
1839        qunlock(&d->portm.ql);
1840        return i;
1841}
1842
1843static int iariopkt(struct sdreq *r, struct drive *d)
1844{
1845        ERRSTACK(2);
1846        int n, count, try
1847                , max, flag, task, wormwrite;
1848        char *name;
1849        unsigned char *cmd, *data;
1850        void *port;
1851        struct Asleep as;
1852
1853        cmd = r->cmd;
1854        name = d->unit->sdperm.name;
1855        port = d->port;
1856
1857        aprintd("ahci: iariopkt: %04#x %04#x %c %d %p\n", cmd[0], cmd[2],
1858                "rw"[r->write], r -> dlen, r -> data);
1859        if (cmd[0] == 0x5a && (cmd[2] & 0x3f) == 0x3f)
1860                return sdmodesense(r, cmd, d->info, d->infosz);
1861        r->rlen = 0;
1862        count = r->dlen;
1863        max = 65536;
1864
1865        try
1866                = 0;
1867retry:
1868        data = r->data;
1869        n = count;
1870        if (n > max)
1871                n = max;
1872        ahcibuildpkt(&d->portm, r, data, n);
1873        switch (waitready(d)) {
1874        case -1:
1875                qunlock(&d->portm.ql);
1876                return SDeio;
1877        case 1:
1878                qunlock(&d->portm.ql);
1879                esleep(1);
1880                goto retry;
1881        }
1882        /* d->portm qlock held here */
1883
1884        spin_lock_irqsave(&d->Lock);
1885        d->portm.flag = 0;
1886        spin_unlock_irqsave(&d->Lock);
1887        ahci_port_write32(port, PORT_CI, 1);
1888
1889        as.p = port;
1890        as.i = 1;
1891        d->intick = ms();
1892        d->active++;
1893
1894        while (waserror())
1895                poperror();
1896        /* don't sleep here forever */
1897        rendez_sleep_timeout(&d->portm.Rendez, ahciclear, &as,
1898                             (3 * 1000) * 1000);
1899        poperror();
1900        if (!ahciclear(&as)) {
1901                qunlock(&d->portm.ql);
1902                printd("%s: ahciclear not true after 3 seconds\n", name);
1903                r->status = SDcheck;
1904                return SDcheck;
1905        }
1906
1907        d->active--;
1908        spin_lock_irqsave(&d->Lock);
1909        flag = d->portm.flag;
1910        task = ahci_port_read32(port, PORT_TFD);
1911        spin_unlock_irqsave(&d->Lock);
1912
1913        if ((task & (Efatal << 8)) ||
1914            ((task & (ASbsy | ASdrq)) && (d->state == Dready))) {
1915                ahci_port_write32(port, PORT_CI, 0);
1916                ahcirecover(&d->portc);
1917                task = ahci_port_read32(port, PORT_TFD);
1918                flag &= ~Fdone; /* either an error or do-over */
1919        }
1920        qunlock(&d->portm.ql);
1921        if (flag == 0) {
1922                if (++try == 10) {
1923                        printd("%s: bad disk\n", name);
1924                        r->status = SDcheck;
1925                        return SDcheck;
1926                }
1927                /*
1928                 * write retries cannot succeed on write-once media,
1929                 * so just accept any failure.
1930                 */
1931                wormwrite = 0;
1932                switch (d->unit->inquiry[0] & SDinq0periphtype) {
1933                case SDperworm:
1934                case SDpercd:
1935                        switch (cmd[0]) {
1936                        case 0x0a: /* write (6?) */
1937                        case 0x2a: /* write (10) */
1938                        case 0x8a: /* int32_t write (16) */
1939                        case 0x2e: /* write and verify (10) */
1940                                wormwrite = 1;
1941                                break;
1942                        }
1943                        break;
1944                }
1945                if (!wormwrite) {
1946                        printd("%s: retry\n", name);
1947                        goto retry;
1948                }
1949        }
1950        if (flag & Ferror) {
1951                if ((task & Eidnf) == 0)
1952                        printd("%s: i/o error task=%#x\n", name, task);
1953                r->status = SDcheck;
1954                return SDcheck;
1955        }
1956
1957        data += n;
1958
1959        r->rlen = data - (unsigned char *)r->data;
1960        r->status = SDok;
1961        return SDok;
1962}
1963
1964static int iario(struct sdreq *r)
1965{
1966        ERRSTACK(1);
1967        int i, n, count, try
1968                , max, flag, task;
1969        uint64_t lba;
1970        char *name;
1971        unsigned char *cmd, *data;
1972        void *port;
1973        struct Asleep as;
1974        struct ctlr *c;
1975        struct drive *d;
1976        struct sdunit *unit;
1977
1978        unit = r->unit;
1979        c = unit->dev->ctlr;
1980        d = c->drive[unit->subno];
1981        if (d->portm.feat & Datapi)
1982                return iariopkt(r, d);
1983        cmd = r->cmd;
1984        name = d->unit->sdperm.name;
1985        port = d->port;
1986
1987        if (r->cmd[0] == 0x35 || r->cmd[0] == 0x91) {
1988                if (flushcache(d) == 0)
1989                        return sdsetsense(r, SDok, 0, 0, 0);
1990                return sdsetsense(r, SDcheck, 3, 0xc, 2);
1991        }
1992
1993        if ((i = sdfakescsi(r, d->info, d->infosz)) != SDnostatus) {
1994                r->status = i;
1995                return i;
1996        }
1997
1998        if (*cmd != 0x28 && *cmd != 0x2a) {
1999                printd("%s: bad cmd %.2#x\n", name, cmd[0]);
2000                r->status = SDcheck;
2001                return SDcheck;
2002        }
2003
2004        // TODO: make cmd bigger to support drives with >= 2 TiB capacity,
2005        // with 32 bits and 512 B blocks only 2^(9+32) = 2 TiB addressable
2006        lba = (uint32_t)(cmd[2] << 24) | cmd[3] << 16 | cmd[4] << 8 | cmd[5];
2007        count = cmd[7] << 8 | cmd[8];
2008        if (r->data == NULL)
2009                return SDok;
2010        if (r->dlen < count * unit->secsize)
2011                count = r->dlen / unit->secsize;
2012        max = 128;
2013
2014        try
2015                = 0;
2016retry:
2017        data = r->data;
2018        while (count > 0) {
2019                n = count;
2020                if (n > max)
2021                        n = max;
2022                ahcibuild(d, cmd, data, n, lba);
2023                switch (waitready(d)) {
2024                case -1:
2025                        qunlock(&d->portm.ql);
2026                        return SDeio;
2027                case 1:
2028                        qunlock(&d->portm.ql);
2029                        esleep(1);
2030                        goto retry;
2031                }
2032                /* d->portm qlock held here */
2033                spin_lock_irqsave(&d->Lock);
2034                d->portm.flag = 0;
2035                spin_unlock_irqsave(&d->Lock);
2036                ahci_port_write32(port, PORT_CI, 1);
2037
2038                as.p = port;
2039                as.i = 1;
2040                d->intick = ms();
2041                d->active++;
2042
2043                while (waserror())
2044                        poperror();
2045                /* don't sleep here forever */
2046                rendez_sleep_timeout(&d->portm.Rendez, ahciclear, &as,
2047                                     (3 * 1000) * 1000);
2048                poperror();
2049                if (!ahciclear(&as)) {
2050                        qunlock(&d->portm.ql);
2051                        printd("%s: ahciclear not true after 3 seconds\n",
2052                               name);
2053                        r->status = SDcheck;
2054                        return SDcheck;
2055                }
2056
2057                d->active--;
2058                spin_lock_irqsave(&d->Lock);
2059                flag = d->portm.flag;
2060                task = ahci_port_read32(port, PORT_TFD);
2061                spin_unlock_irqsave(&d->Lock);
2062
2063                if ((task & (Efatal << 8)) ||
2064                    ((task & (ASbsy | ASdrq)) && d->state == Dready)) {
2065                        ahci_port_write32(port, PORT_CI, 0);
2066                        ahcirecover(&d->portc);
2067                        task = ahci_port_read32(port, PORT_TFD);
2068                }
2069                qunlock(&d->portm.ql);
2070                if (flag == 0) {
2071                        if (++try == 10) {
2072                                printd("%s: bad disk\n", name);
2073                                r->status = SDeio;
2074                                return SDeio;
2075                        }
2076                        printd("%s: retry blk %lld\n", name, lba);
2077                        goto retry;
2078                }
2079                if (flag & Ferror) {
2080                        printk("%s: i/o error task=%#x @%lld\n", name, task,
2081                               lba);
2082                        r->status = SDeio;
2083                        return SDeio;
2084                }
2085
2086                count -= n;
2087                lba += n;
2088                data += n * unit->secsize;
2089        }
2090        r->rlen = data - (unsigned char *)r->data;
2091        r->status = SDok;
2092        return SDok;
2093}
2094
2095/*
2096 * configure drives 0-5 as ahci sata (c.f. errata).
2097 * what about 6 & 7, as claimed by marvell 0x9123?
2098 */
2099static int iaahcimode(struct pci_device *p)
2100{
2101        printd("iaahcimode: %#x %#x %#x\n", pcidev_read8(p, 0x91),
2102               pcidev_read8(p, 92), pcidev_read8(p, 93));
2103        pcidev_write16(p, 0x92, pcidev_read16(p, 0x92) | 0x3f); /* ports 0-5 */
2104        return 0;
2105}
2106
2107static void iasetupahci(struct ctlr *c)
2108{
2109        void *p = c->mmio; /* This is actually a pointer to HBA */
2110        /* disable cmd block decoding. */
2111        pcidev_write16(c->pci, 0x40, pcidev_read16(c->pci, 0x40) & ~(1 << 15));
2112        pcidev_write16(c->pci, 0x42, pcidev_read16(c->pci, 0x42) & ~(1 << 15));
2113
2114        ahci_hba_write32(p, HBA_GHC,
2115                         1 << 31); /* enable ahci mode (ghc register) */
2116        ahci_hba_write32(p, HBA_PI,
2117                         (1 << 6) - 1); /* 5 ports. (supposedly ro pi reg.) */
2118
2119        /* enable ahci mode and 6 ports; from ich9 datasheet */
2120        pcidev_write16(c->pci, 0x90, 1 << 6 | 1 << 5);
2121}
2122
2123static int didtype(struct pci_device *p)
2124{
2125        switch (p->ven_id) {
2126        case Vintel:
2127                if ((p->dev_id & 0xfffc) == 0x2680)
2128                        return Tesb;
2129                /*
2130                 * 0x27c4 is the intel 82801 in compatibility (not sata) mode.
2131                 */
2132                if (p->dev_id == 0x1e02 ||            /* c210 */
2133                    p->dev_id == 0x24d1 ||            /* 82801eb/er */
2134                    (p->dev_id & 0xfffb) == 0x27c1 || /* 82801g[bh]m ich7 */
2135                    p->dev_id == 0x2821 ||            /* 82801h[roh] */
2136                    (p->dev_id & 0xfffe) == 0x2824 || /* 82801h[b] */
2137                    (p->dev_id & 0xfeff) == 0x2829 || /* ich8/9m */
2138                    (p->dev_id & 0xfffe) == 0x2922 || /* ich9 */
2139                    p->dev_id == 0x3a02 ||            /* 82801jd/do */
2140                    (p->dev_id & 0xfefe) == 0x3a22 || /* ich10, pch */
2141                    (p->dev_id & 0xfff8) == 0x3b28 || /* pchm */
2142                    p->dev_id == 0x1d02)              /* c600/x79 pch */
2143                        return Tich;
2144                break;
2145        case Vatiamd:
2146                if (p->dev_id == 0x4380 || p->dev_id == 0x4390 ||
2147                    p->dev_id == 0x4391) {
2148                        printd("detected sb600 vid %#x did %#x\n", p->ven_id,
2149                               p->dev_id);
2150                        return Tsb600;
2151                }
2152                break;
2153        case Vmarvell:
2154                if (p->dev_id == 0x9123)
2155                        printk("ahci: marvell sata 3 controller has delusions "
2156                               "of something on unit 7\n");
2157                break;
2158        }
2159        if (p->class == Pcibcstore && p->subclass == Pciscsata &&
2160            p->progif == 1) {
2161                printd("ahci: Tunk: vid %#4.4x did %#4.4x\n", p->ven_id,
2162                       p->dev_id);
2163                return Tunk;
2164        }
2165        return -1;
2166}
2167
2168static int newctlr(struct ctlr *ctlr, struct sdev *sdev, int nunit)
2169{
2170        int i, n;
2171        struct drive *drive;
2172        uint32_t h_cap, pi;
2173
2174        ctlr->ndrive = sdev->nunit = nunit;
2175        h_cap = ahci_hba_read32(ctlr->hba, HBA_CAP);
2176        printd("ahci: %s: HBA_CAP=0x%08x\n", __func__, h_cap);
2177        ctlr->mport = h_cap & ((1 << 5) - 1);
2178
2179        i = (h_cap >> 20) & ((1 << 4) - 1); /* iss */
2180        printk("#S/sd%c: %s: %s, %d ports, irq %d\n", sdev->idno,
2181               Tname(ctlr), descmode[i], nunit, ctlr->pci->irqline);
2182        /* map the drives -- they don't all need to be enabled. */
2183        n = 0;
2184        ctlr->rawdrive = kzmalloc(NCtlrdrv * sizeof(struct drive), 0);
2185        if (ctlr->rawdrive == NULL) {
2186                printd("ahci: out of memory\n");
2187                return -1;
2188        }
2189        pi = ahci_hba_read32(ctlr->hba, HBA_PI);
2190        for (i = 0; i < NCtlrdrv; i++) {
2191                drive = ctlr->rawdrive + i;
2192                spinlock_init_irqsave(&drive->Lock);
2193                drive->portno = i;
2194                drive->driveno = -1;
2195                drive->sectors = 0;
2196                drive->serial[0] = ' ';
2197                drive->ctlr = ctlr;
2198                if ((pi & (1 << i)) == 0)
2199                        continue;
2200                drive->port = ctlr->mmio + 0x80 * i + 0x100;
2201                drive->portc.p = drive->port;
2202                drive->portc.pm = &drive->portm;
2203                qlock_init(&drive->portm.ql);
2204                rendez_init(&drive->portm.Rendez);
2205                drive->driveno = n++;
2206                ctlr->drive[drive->driveno] = drive;
2207                iadrive[niadrive + drive->driveno] = drive;
2208        }
2209        for (i = 0; i < n; i++)
2210                if (ahciidle(ctlr->drive[i]->port) == -1) {
2211                        printd("ahci: %s: port %d wedged; abort\n", Tname(ctlr),
2212                               i);
2213                        return -1;
2214                }
2215        for (i = 0; i < n; i++) {
2216                ctlr->drive[i]->mode = DMsatai;
2217                configdrive(ctlr->drive[i]);
2218        }
2219        return n;
2220}
2221
2222static void releasedrive(struct kref *kref)
2223{
2224        printk("release drive called, but we don't do that yet\n");
2225}
2226
2227static struct sdev *iapnp(void)
2228{
2229        int n, nunit, type;
2230        struct ctlr *c;
2231        struct pci_device *p;
2232        struct sdev *head, *tail, *s;
2233
2234        // TODO: ensure we're only called once.
2235
2236        memset(olds, 0xff, sizeof olds);
2237        p = NULL;
2238        head = tail = NULL;
2239        STAILQ_FOREACH (p, &pci_devices, all_dev) {
2240                type = didtype(p);
2241                if (type == -1)
2242                        continue;
2243                printd("ahci: %s: ven_id=0x%04x, dev_id=0x%04x, didtype=%d\n",
2244                       __func__, p->ven_id, p->dev_id, type);
2245                /* Not sure if this check matters, or if it was code cruft.
2246                 * Why wouldn't a 64 bit BAR work? */
2247                if (!pci_bar_is_mem32(p, Abar)) {
2248                        printk("ahci: bar %d was not 'mem32' - aborting\n",
2249                               Abar);
2250                        continue;
2251                }
2252                if (niactlr == NCtlr) {
2253                        printk("ahci: iapnp: %s: too many controllers\n",
2254                               tname[type]);
2255                        break;
2256                }
2257                c = iactlr + niactlr;
2258                s = sdevs + niactlr;
2259                memset(c, 0, sizeof *c);
2260                memset(s, 0, sizeof *s);
2261                kref_init(&s->r, releasedrive, 1);
2262                qlock_init(&s->ql);
2263                qlock_init(&s->unitlock);
2264                spinlock_init_irqsave(&c->Lock);
2265                c->mmio = pci_get_mmio_bar_kva(p, Abar);
2266                if (c->mmio == 0) {
2267                        printk("ahci: %s: could not map bar %d did=%#x\n",
2268                               Tname(c), Abar, p->dev_id);
2269                        continue;
2270                }
2271                c->pci = p;
2272                c->type = type;
2273
2274                s->ifc = &sdiahciifc;
2275                s->idno = 'E' + niactlr;
2276                s->ctlr = c;
2277                c->sdev = s;
2278
2279                if (Intel(c) && p->dev_id != 0x2681)
2280                        iasetupahci(c);
2281                nunit = ahciconf(c);
2282                // ahcihbareset((void *)c->mmio);
2283                if (Intel(c) && iaahcimode(p) == -1)
2284                        break;
2285                if (nunit < 1)
2286                        continue;
2287                n = newctlr(c, s, nunit);
2288                if (n < 0)
2289                        continue;
2290                niadrive += n;
2291                niactlr++;
2292                if (head)
2293                        tail->next = s;
2294                else
2295                        head = s;
2296                tail = s;
2297        }
2298        return head;
2299}
2300
2301static char *smarttab[] = {"unset", "error", "threshold exceeded", "normal"};
2302
2303static char *pflag(char *s, char *e, unsigned char f)
2304{
2305        unsigned char i;
2306
2307        for (i = 0; i < 8; i++)
2308                if (f & (1 << i))
2309                        s = seprintf(s, e, "%s ", flagname[i]);
2310        return seprintf(s, e, "\n");
2311}
2312
2313static int iarctl(struct sdunit *u, char *p, int l)
2314{
2315        char buf[32];
2316        char *e, *op;
2317        void *port;
2318        struct ctlr *c;
2319        struct drive *d;
2320        uint32_t serror, task, cmd, ci, is, sig, sstatus;
2321
2322        c = u->dev->ctlr;
2323        if (c == NULL) {
2324                printk("iarctl: nil u->dev->ctlr\n");
2325                return 0;
2326        }
2327        d = c->drive[u->subno];
2328        port = d->port;
2329
2330        e = p + l;
2331        op = p;
2332        if (d->state == Dready) {
2333                p = seprintf(p, e, "model\t%s\n", d->model);
2334                p = seprintf(p, e, "serial\t%s\n", d->serial);
2335                p = seprintf(p, e, "firm\t%s\n", d->firmware);
2336                if (d->smartrs == 0xff)
2337                        p = seprintf(p, e, "smart\tenable error\n");
2338                else if (d->smartrs == 0)
2339                        p = seprintf(p, e, "smart\tdisabled\n");
2340                else
2341                        p = seprintf(p, e, "smart\t%s\n",
2342                                     smarttab[d->portm.smart]);
2343                p = seprintf(p, e, "flag\t");
2344                p = pflag(p, e, d->portm.feat);
2345        } else
2346                p = seprintf(p, e, "no disk present [%s]\n",
2347                             diskstates[d->state]);
2348        serror = ahci_port_read32(port, PORT_SERR);
2349        task = ahci_port_read32(port, PORT_TFD);
2350        cmd = ahci_port_read32(port, PORT_CMD);
2351        ci = ahci_port_read32(port, PORT_CI);
2352        is = ahci_port_read32(port, PORT_IS);
2353        sig = ahci_port_read32(port, PORT_SIG);
2354        sstatus = ahci_port_read32(port, PORT_SSTS);
2355        serrstr(serror, buf, buf + sizeof(buf) - 1);
2356        p = seprintf(p, e,
2357                     "reg\ttask %#lx cmd %#lx serr %#lx %s ci %#lx is %#lx; "
2358                     "sig %#lx sstatus %06#lx\n",
2359                     task, cmd, serror, buf, ci, is, sig, sstatus);
2360        if (d->unit == NULL)
2361                panic("iarctl: nil d->unit");
2362        p = seprintf(p, e, "geometry %llu %lu\n", d->sectors, d->unit->secsize);
2363        return p - op;
2364}
2365
2366static void runflushcache(struct drive *d)
2367{
2368        int32_t t0;
2369
2370        t0 = ms();
2371        if (flushcache(d) != 0)
2372                error(EIO, "Flush cache failed");
2373        printd("ahci: flush in %ld ms\n", ms() - t0);
2374}
2375
2376static void forcemode(struct drive *d, char *mode)
2377{
2378        int i;
2379
2380        for (i = 0; i < ARRAY_SIZE(modename); i++)
2381                if (strcmp(mode, modename[i]) == 0)
2382                        break;
2383        if (i == ARRAY_SIZE(modename))
2384                i = 0;
2385        spin_lock_irqsave(&d->Lock);
2386        d->mode = i;
2387        spin_unlock_irqsave(&d->Lock);
2388}
2389
2390static void runsmartable(struct drive *d, int i)
2391{
2392        ERRSTACK(1);
2393
2394        if (waserror()) {
2395                qunlock(&d->portm.ql);
2396                d->smartrs = 0;
2397                nexterror();
2398        }
2399        if (lockready(d) == -1)
2400                error(EIO, "runsmartable: lockready returned -1");
2401        d->smartrs = smart(&d->portc, i);
2402        d->portm.smart = 0;
2403        qunlock(&d->portm.ql);
2404        poperror();
2405}
2406
2407static void forcestate(struct drive *d, char *state)
2408{
2409        int i;
2410
2411        for (i = 0; i < ARRAY_SIZE(diskstates); i++)
2412                if (strcmp(state, diskstates[i]) == 0)
2413                        break;
2414        if (i == ARRAY_SIZE(diskstates))
2415                error(EINVAL, "Can't set state to invalid value '%s'", state);
2416        setstate(d, i);
2417}
2418
2419/*
2420 * force this driver to notice a change of medium if the hardware doesn't
2421 * report it.
2422 */
2423static void changemedia(struct sdunit *u)
2424{
2425        struct ctlr *c;
2426        struct drive *d;
2427
2428        c = u->dev->ctlr;
2429        d = c->drive[u->subno];
2430        spin_lock_irqsave(&d->Lock);
2431        d->mediachange = 1;
2432        u->sectors = 0;
2433        spin_unlock_irqsave(&d->Lock);
2434}
2435
2436static int iawctl(struct sdunit *u, struct cmdbuf *cmd)
2437{
2438        ERRSTACK(1);
2439        char **f;
2440        struct ctlr *c;
2441        struct drive *d;
2442        unsigned int i;
2443
2444        c = u->dev->ctlr;
2445        d = c->drive[u->subno];
2446        f = cmd->f;
2447
2448        if (strcmp(f[0], "change") == 0)
2449                changemedia(u);
2450        else if (strcmp(f[0], "flushcache") == 0)
2451                runflushcache(d);
2452        else if (strcmp(f[0], "identify") == 0) {
2453                i = strtoul(f[1] ? f[1] : "0", 0, 0);
2454                if (i > 0xff)
2455                        i = 0;
2456                printd("ahci: %04d %#x\n", i, d->info[i]);
2457        } else if (strcmp(f[0], "mode") == 0)
2458                forcemode(d, f[1] ? f[1] : "satai");
2459        else if (strcmp(f[0], "nop") == 0) {
2460                if ((d->portm.feat & Dnop) == 0) {
2461                        sdierror(cmd, "no drive support");
2462                        return -1;
2463                }
2464                if (waserror()) {
2465                        qunlock(&d->portm.ql);
2466                        nexterror();
2467                }
2468                if (lockready(d) == -1)
2469                        error(EIO, "%s: lockready returned -1", __func__);
2470                nop(&d->portc);
2471                qunlock(&d->portm.ql);
2472                poperror();
2473        } else if (strcmp(f[0], "reset") == 0)
2474                forcestate(d, "reset");
2475        else if (strcmp(f[0], "smart") == 0) {
2476                if (d->smartrs == 0)
2477                        sdierror(cmd, "smart not enabled");
2478                if (waserror()) {
2479                        qunlock(&d->portm.ql);
2480                        d->smartrs = 0;
2481                        nexterror();
2482                }
2483                if (lockready(d) == -1)
2484                        error(EIO, "%s: lockready returned -1", __func__);
2485                d->portm.smart = 2 + smartrs(&d->portc);
2486                qunlock(&d->portm.ql);
2487                poperror();
2488        } else if (strcmp(f[0], "smartdisable") == 0)
2489                runsmartable(d, 1);
2490        else if (strcmp(f[0], "smartenable") == 0)
2491                runsmartable(d, 0);
2492        else if (strcmp(f[0], "state") == 0)
2493                forcestate(d, f[1] ? f[1] : "null");
2494        else {
2495                sdierror(cmd, "%s: unknown control '%s'", __func__, f[0]);
2496                return -1;
2497        }
2498        return 0;
2499}
2500
2501static char *portr(char *p, char *e, unsigned int x)
2502{
2503        int i, a;
2504
2505        p[0] = 0;
2506        a = -1;
2507        for (i = 0; i < 32; i++) {
2508                if ((x & (1 << i)) == 0) {
2509                        if (a != -1 && i - 1 != a)
2510                                p = seprintf(p, e, "-%d", i - 1);
2511                        a = -1;
2512                        continue;
2513                }
2514                if (a == -1) {
2515                        if (i > 0)
2516                                p = seprintf(p, e, ", ");
2517                        p = seprintf(p, e, "%d", a = i);
2518                }
2519        }
2520        if (a != -1 && i - 1 != a)
2521                p = seprintf(p, e, "-%d", i - 1);
2522        return p;
2523}
2524
2525/* must emit exactly one line per controller (sd(3)) */
2526static char *iartopctl(struct sdev *sdev, char *p, char *e)
2527{
2528        uint32_t cap, ghc, isr, pi, ver;
2529        char pr[25];
2530        void *hba;
2531        struct ctlr *ctlr;
2532
2533#define has(x, str)                                                            \
2534        do {                                                                   \
2535                if (cap & (x))                                                 \
2536                        p = seprintf(p, e, "%s ", (str));                      \
2537        } while (0)
2538
2539        ctlr = sdev->ctlr;
2540        hba = ctlr->hba;
2541        p = seprintf(p, e, "sd%c ahci: ", sdev->idno);
2542        cap = ahci_hba_read32(hba, HBA_CAP);
2543        has(Hs64a, "64a");
2544        has(Hsalp, "alp");
2545        has(Hsam, "am");
2546        has(Hsclo, "clo");
2547        has(Hcccs, "coal");
2548        has(Hems, "ems");
2549        has(Hsal, "led");
2550        has(Hsmps, "mps");
2551        has(Hsncq, "ncq");
2552        has(Hssntf, "ntf");
2553        has(Hspm, "pm");
2554        has(Hpsc, "pslum");
2555        has(Hssc, "slum");
2556        has(Hsss, "ss");
2557        has(Hsxs, "sxs");
2558        pi = ahci_hba_read32(hba, HBA_PI);
2559        portr(pr, pr + sizeof(pr), pi);
2560
2561        ghc = ahci_hba_read32(hba, HBA_GHC);
2562        isr = ahci_hba_read32(hba, HBA_ISR);
2563        ver = ahci_hba_read32(hba, HBA_VS);
2564        return seprintf(
2565            p, e,
2566            "iss %ld ncs %ld np %ld; ghc %#lx isr %#lx pi %#lx %s ver %#lx\n",
2567            (cap >> 20) & 0xf, (cap >> 8) & 0x1f, 1 + (cap & 0x1f), ghc, isr,
2568            pi, ver);
2569#undef has
2570}
2571
2572static int iawtopctl(struct sdev *sdev, struct cmdbuf *cmd)
2573{
2574        int *v;
2575        char **f;
2576
2577        f = cmd->f;
2578        v = 0;
2579
2580        if (f[0] == NULL)
2581                return 0;
2582        if (strcmp(f[0], "debug") == 0)
2583                v = &debug;
2584        else if (strcmp(f[0], "iprintd") == 0)
2585                v = &prid;
2586        else if (strcmp(f[0], "aprint") == 0)
2587                v = &datapi;
2588        else
2589                sdierror(cmd, "%s: bad control '%s'", __func__, f[0]);
2590
2591        switch (cmd->nf) {
2592        default:
2593                sdierror(cmd, "%s: %d args, only 1 or 2 allowed", __func__,
2594                         cmd->nf);
2595        case 1:
2596                *v ^= 1;
2597                break;
2598        case 2:
2599                if (f[1])
2600                        *v = strcmp(f[1], "on") == 0;
2601                else
2602                        *v ^= 1;
2603                break;
2604        }
2605        return 0;
2606}
2607
2608struct sdifc sdiahciifc = {
2609    "iahci",
2610
2611    iapnp,     NULL, /* legacy */
2612    iaenable,  iadisable,
2613
2614    iaverify,  iaonline,  iario, iarctl, iawctl,
2615
2616    scsibio,   NULL, /* probe */
2617    NULL,            /* clear */
2618    iartopctl, iawtopctl,
2619};
2620